From 405e209a0bec2dd72efba8079e3ff62db9ffee52 Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Mon, 9 Aug 2021 18:31:58 +0300 Subject: [PATCH 001/344] Command line tool for generating test handler code from templates (#275) Co-authored-by: Shimrit peretz Co-authored-by: Salaheddine Hamadi --- .gitignore | 2 +- DEVELOPING.md | 8 ++ Makefile | 4 + cmd/tnf/main.go | 107 ++++++++++++++++++ go.mod | 1 + go.sum | 1 + pkg/tnf/handlers/handler_template/doc.tmpl | 16 +++ .../handlers/handler_template/handler.tmpl | 80 +++++++++++++ .../handler_template/handler_test.tmpl | 60 ++++++++++ 9 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 cmd/tnf/main.go create mode 100644 pkg/tnf/handlers/handler_template/doc.tmpl create mode 100644 pkg/tnf/handlers/handler_template/handler.tmpl create mode 100644 pkg/tnf/handlers/handler_template/handler_test.tmpl diff --git a/.gitignore b/.gitignore index e8f036a10..fadf74116 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ pkg/tnf/mocks/mock_tester.go pkg/tnf/reel/mocks/mock_reel.go # NOTE: mock_expect.go is tracked, as it is generated from source externally. pkg/tnf/interactive/mocks/mock_spawner.go -temp/ +temp/ \ No newline at end of file diff --git a/DEVELOPING.md b/DEVELOPING.md index 208fa415d..fbb620119 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -604,6 +604,14 @@ The same result could be achieved using a Go Template: oc get pod %s -n %s -o go-template='{{len .spec.containers}}{{"\n"}}' ``` +## Adding new handler + +To facilitate adding new handlers, the developers can leverage existing infrastructure to generate repetitive code. +As an example, to generate handler with name Xyz, the command generate handler Xyz can be used(we need to run this command +in the folder "cmd/tnf/" and we run the command directly here). +The generated code has a template and creates the necessary headers. +The result is folder "new handler" located in /pkg/tnf/handlers/"new handler" that includes 3 files by handler template. + ## Adding information to claim file The result of each test execution is included in the claim file. diff --git a/Makefile b/Makefile index 3b54ad58c..ed5d6d105 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,10 @@ build: make build-cnf-tests make build-jsontest-cli make build-gradetool + make build-tnf-tool + +build-tnf-tool: + go build -o tnf -v cmd/tnf/main.go # (Re)generate mock files as needed mocks: pkg/tnf/interactive/mocks/mock_spawner.go \ diff --git a/cmd/tnf/main.go b/cmd/tnf/main.go new file mode 100644 index 000000000..888a12a7f --- /dev/null +++ b/cmd/tnf/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "bufio" + "log" + "os" + "path" + "text/template" + + "github.com/spf13/cobra" +) + +type myHandler struct { + Handlername string +} + +var ( + pathrelativetoroot string + handlerDirectory string + handlername string + + rootCmd = &cobra.Command{ + Use: "tnf", + Short: "A CLI for creating, validating, and test-network-function tests.", + } + + generate = &cobra.Command{ + Use: "generate", + Short: "generator tool for various tnf artifacts.", + } + + handler = &cobra.Command{ + Use: "handler", + Short: "adding new handler.", + RunE: generateHandlerFiles, + } +) + +func generateHandlerFiles(cmd *cobra.Command, args []string) error { + handlername = args[0] + pathrelativetoroot = path.Join("..", "..") + handlerDirectory = path.Join(pathrelativetoroot, "pkg", "tnf", "handlers") + newHandlerDirectory := path.Join(handlerDirectory, handlername) + + err := os.Mkdir(newHandlerDirectory, 0755) + if err != nil { + return err + } + + myhandler := myHandler{Handlername: handlername} + + // pathfile this is the path of the file from template file that will creat + + pathfile := path.Join(handlerDirectory, "handler_template", "doc.tmpl") + namefile := "" + "doc.go" + err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by doc.tmpl + if err != nil { + return err + } + + pathfile = path.Join(handlerDirectory, "handler_template", "handler_test.tmpl") + namefile = "" + handlername + "_test.go" + err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template_test.tmpl + if err != nil { + return err + } + + pathfile = path.Join(handlerDirectory, "handler_template", "handler.tmpl") + namefile = "" + handlername + ".go" + err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template.tmpl + if err != nil { + return err + } + return err +} + +func createfile(pathfile, namefile string, myhandler myHandler, newHandlerDirectory string) error { + ftpl, err := template.ParseFiles(pathfile) + if err != nil { + return err + } + + temp := path.Join(newHandlerDirectory, namefile) + f, err := os.Create(temp) + if err != nil { + return err + } + + defer f.Close() + w := bufio.NewWriter(f) + + err = ftpl.Execute(w, myhandler) + if err != nil { + return err + } + w.Flush() + + return nil +} + +func main() { + rootCmd.AddCommand(generate) + generate.AddCommand(handler) + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/go.mod b/go.mod index 76a70ccc7..df5724bc0 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.39.0 diff --git a/go.sum b/go.sum index 3f0a4e724..7b91fff22 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= diff --git a/pkg/tnf/handlers/handler_template/doc.tmpl b/pkg/tnf/handlers/handler_template/doc.tmpl new file mode 100644 index 000000000..758d74831 --- /dev/null +++ b/pkg/tnf/handlers/handler_template/doc.tmpl @@ -0,0 +1,16 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package {{ .Handlername }} \ No newline at end of file diff --git a/pkg/tnf/handlers/handler_template/handler.tmpl b/pkg/tnf/handlers/handler_template/handler.tmpl new file mode 100644 index 000000000..48fabf222 --- /dev/null +++ b/pkg/tnf/handlers/handler_template/handler.tmpl @@ -0,0 +1,80 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package {{ .Handlername }} + +import ( + "time" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +// TODO: +type {{ .Handlername }} struct { + result int + timeout time.Duration + args []string + // adding special parameters +} + +const ( + // adding special variables +) + +func (h *{{ .Handlername }}) Args() []string { + return h.args +} + +// GetIdentifier returns the tnf.Test specific identifier. +func (h *{{ .Handlername }}) GetIdentifier() { + // TODO : create identifier.{{ .Handlername}}Identifier. +} + +// Timeout return the timeout for the test. +func (h *{{ .Handlername }}) Timeout() time.Duration { + return h.timeout +} + +// Result returns the test result. +func (h *{{ .Handlername }}) Result() int { + return h.result +} + +// ReelFirst returns a step which expects an Handlername summary for the given device. +func (h *{{ .Handlername }}) ReelFirst() *reel.Step { + return &reel.Step{ + Expect: []string{""}, // TODO : pass the list of possible regex in here + Timeout: h.timeout, + } +} + +// ReelMatch parses the Handlername output and set the test result on match. +func (h *{{ .Handlername }}) ReelMatch(_, _, match string) *reel.Step { + h.result = tnf.ERROR + return nil + // TODO : add the matching logic in here +} + +// ReelTimeout does nothing; Handlername requires no explicit intervention for a timeout. +func (h *{{ .Handlername }}) ReelTimeout() *reel.Step { + return nil + // TODO : fill the stub +} + +// ReelEOF does nothing; Handlername requires no explicit intervention for EOF. +func (h *{{ .Handlername }}) ReelEOF() { + // TODO : fill the stub +} diff --git a/pkg/tnf/handlers/handler_template/handler_test.tmpl b/pkg/tnf/handlers/handler_template/handler_test.tmpl new file mode 100644 index 000000000..6597c4a17 --- /dev/null +++ b/pkg/tnf/handlers/handler_template/handler_test.tmpl @@ -0,0 +1,60 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package {{ .Handlername }} + +import ( + "testing" +) + +const ( + // adding special variable +) + +// {{ .Handlername }}_Args() +func Test{{ .Handlername }}_Args(){ + // Todo : write test for function {{ .Handlername }}_Args() +} + +// {{ .Handlername }}_GetIdentifier +func Test{{ .Handlername }}_GetIdentifier(){ + // Todo : write test for function {{ .Handlername }}_GetIdentifier +} + +// {{ .Handlername }}_ReelFirst +func Test{{ .Handlername }}_ReelFirst(){ + // Todo : write test for function {{ .Handlername }}_ReelFirst +} + +// {{ .Handlername }}_ReelEof +func Test{{ .Handlername }}_ReelEof(){ + // Todo : write test for function {{ .Handlername }}_ReelEof +} + +// {{ .Handlername }}_ReelTimeout +func Test{{ .Handlername }}_ReelTimeout(){ + // Todo : write test for function {{ .Handlername }}_ReelTimeout +} + +// {{ .Handlername }}_ReelMatch +func Test{{ .Handlername }}_ReelMatch(){ + // Todo : write test for function {{ .Handlername }}_ReelMatch +} + +// New{{ .Handlername }} +func TestNew{{ .Handlername }}(t *testing.T){ + // Todo : write test for function TestNew{{ .Handlername }} +} \ No newline at end of file From 15777e347068943704eafa328b20d86311943fb1 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Mon, 9 Aug 2021 17:22:12 -0400 Subject: [PATCH 002/344] Config package refactoring and cleanup (#293) Config package refactoring and cleanup --- .gitignore | 2 + README.md | 130 +++++++--------- docs/config.md | 59 -------- examples/example_config.yaml | 77 ---------- pkg/config/autodiscover/autodiscover.go | 60 ++++++++ pkg/config/autodiscover/autodiscover_cnfs.go | 77 ---------- .../autodiscover/autodiscover_generic.go | 134 ----------------- .../autodiscover/autodiscover_operator.go | 85 ----------- .../autodiscover/autodiscover_partner.go | 51 +++++++ .../autodiscover/autodiscover_targets.go | 142 ++++++++++++++++++ pkg/config/autodiscover/container_test.go | 39 +++-- pkg/config/autodiscover/csv_info_test.go | 5 +- pkg/config/autodiscover/generic_test.go | 52 ------- pkg/config/autodiscover/operator_test.go | 5 +- pkg/config/autodiscover/pod_info_test.go | 5 +- pkg/config/autodiscover/pod_test.go | 41 +++++ .../autodiscover/testdata/testtarget.json | 2 +- pkg/config/config.go | 102 +------------ pkg/config/config_test.go | 6 +- .../configsections/certified_request_test.go | 28 ++-- pkg/config/configsections/common.go | 53 +++++++ ...ntainer_test.go => config_section_test.go} | 58 +++---- .../{generic_config.go => container.go} | 10 -- .../{container_config.go => misc.go} | 52 +------ pkg/config/configsections/pod.go | 29 ++++ pkg/config/configsections/request.go | 37 +++++ pkg/config/testdata/tnf_test_config.yml | 33 ++-- pkg/tnf/handlers/container/pod.go | 4 +- pkg/tnf/testcases/base.go | 24 +-- pkg/tnf/testcases/base_test.go | 2 +- test-network-function/accesscontrol/suite.go | 53 ++++--- test-network-function/common/env.go | 2 +- test-network-function/testconfigure.yml | 1 - test-network-function/tnf_config.yml | 88 ++++++----- 34 files changed, 665 insertions(+), 883 deletions(-) delete mode 100644 docs/config.md delete mode 100644 examples/example_config.yaml delete mode 100644 pkg/config/autodiscover/autodiscover_cnfs.go delete mode 100644 pkg/config/autodiscover/autodiscover_generic.go delete mode 100644 pkg/config/autodiscover/autodiscover_operator.go create mode 100644 pkg/config/autodiscover/autodiscover_partner.go create mode 100644 pkg/config/autodiscover/autodiscover_targets.go delete mode 100644 pkg/config/autodiscover/generic_test.go create mode 100644 pkg/config/autodiscover/pod_test.go rename pkg/config/configsections/{container_test.go => config_section_test.go} (82%) rename pkg/config/configsections/{generic_config.go => container.go} (63%) rename pkg/config/configsections/{container_config.go => misc.go} (52%) create mode 100644 pkg/config/configsections/pod.go create mode 100644 pkg/config/configsections/request.go diff --git a/.gitignore b/.gitignore index fadf74116..6808ee558 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ setup_junit.xml validation_junit.xml .vscode jsontest-cli +gradetool +test-out.json # Explicitly don't track mocks. mocks should be built on request. pkg/tnf/mocks/mock_tester.go diff --git a/README.md b/README.md index 56392d659..32b4f99cc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # Test Network Function ![build](https://github.com/test-network-function/test-network-function/actions/workflows/merge.yaml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/test-network-function/test-network-function)](https://goreportcard.com/report/github.com/test-network-function/test-network-function) @@ -14,54 +15,38 @@ certification. Please see "CNF Developers" below for more information. ![overview](docs/images/overview.svg) In the diagram above: -- the `CNF under test` is the CNF to be certified. The certification suite identifies the pods belonging to the CNF via a label, see [Pod Roles](#pod_roles) +- the `CNF under test` is the CNF to be certified. The certification suite identifies the resources (containers/pods/operators etc) belonging to the CNF via labels or static data entries in the config file - the `Certification container/exec` is the certification test suite running on the platform or in a container. The executable verifies the CNF under test configuration and its interactions with openshift -- the `Partner pod` can be any pod with the required tools (e.g. only ping at the moment) in the same namespace as the `CNF under test`. During connectivity tests, the partner pod will generate pings towards the `CNF under test` to verify connectivity. - -These roles are configured in the [Test Configuration](#test-configuration) section below. +- the `Partner pod` can be any pod with the required tools in the same namespace as the `CNF under test`. For example, during connectivity tests, the partner pod will generate pings towards the `CNF under test` to verify connectivity. The partner pods/containers are auto deployed by the test suite prior a test run and can be auto discovered by the suite without any data entry in the config file. ## Test Configuration -Detailed configuration of the individual specs is explained in [config.md](docs/config.md). - -By leveraging resource labels. Automatic configuration will happen by default, that being said, it can be disabled if the environment variable `TNF_DISABLE_CONFIG_AUTODISCOVER` is set +The Test Network Function support auto-configuration using labels and annotations, but also a static configuration using a file. The following sections describe how to configure the TNF via labels/annotation and the corresponding settings in the config file. A sample config file can be found [here](test-network-function/tnf_config.yml). -```shell -TNF_DISABLE_CONFIG_AUTODISCOVER=true +### targetPodLabels +The goal of this section is to specify the label to be used to identify the cnf under test pods. So for example, with the default configuration: +```shell script +targetPodLabels: + - namespace: test-network-function.com + name: generic + value: target ``` -Pods can be labelled at creation by including the label in their definition, or at any time using the `oc label` -command. Annotations are almost identical: - -```shell -oc label pod test -n tnf test-network-function.com/generic=target -oc annotate pod test -n tnf test-network-function.com/skip_connectivity_tests=true +The corresponding label prefix is: +```shell script +test-network-function.com/generic: target ``` +When labelling a pod to be discovered and tested, the discoered pods are **in addition to** the ones +explicitly configured in the testTarget sections of the config file. -**NOTE** Currently when this environment variable is set the `generic` and `cnfs` sections of the config file will be -ENTIRELY replaced with the autodiscovered configuration, to avoid potentially non-obvious errors. - -### Pod Roles +### testTarget +#### podsUnderTest / containersUnderTest +This section is usually not required if labels defined in the section above cover all resources that should be tested. It's highly recommended that the labels shoud be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. -* The test partner pod is identified by the label `test-network-function.com/generic=orchestrator`. This must identify a -single pod with a single container. This is equivalent to having an entry listed under `partnerContainers` in the config file. -* Each pod under test is identified by the label `test-network-function.com/generic=target`. There must be at least -one such pod. This is equivalent to having a pod listed under `containersUnderTest` in the config file. -* If intrusive test is NOT disabled (refer to [intrusive tests](#disable-intrusive-tests), the pod under test may get -recreated with a different name and lose the "target" label. It's **important** to populate the matching labels from the -deployment in the targetPodLabels section of the config file. -* If an FS Diff Master Pod is present it should be identified with, `test-network-function.com/generic=fs_diff_master`. This -is equivalent to listing the pod under `fsDiffMasterContainer` in the config file. -* If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. `ping`), it should be -given the label `test-network-function.com/skip_connectivity_tests` to exclude it from those tests. The label value is -not important, only its presence. Equivalent to `excludeContainersFromConnectivityTests` in the config file. - -The autodiscovery mechanism will attempt to identify the default network device and all the IP addresses of the pods it -needs, though that information can be explicitly set using annotations if needed. The following rules apply: - -### Pod IPs +The autodiscovery mechanism will also attempt to identify the default network device and all the IP addresses of the pods it +needs for network connectivity tests, though that information can be explicitly set using annotations if needed. For Pod IPs: * The annotation `test-network-function.com/multusips` is the highest priority, and must contain a JSON-encoded list of IP addresses to be tested for the pod. This must be explicitly set. @@ -69,7 +54,7 @@ IP addresses to be tested for the pod. This must be explicitly set. used. This annotation is automatically managed in OpenShift but may not be present in K8s. * If neither of the above is present, then only known IPs associated with the pod are used (the pod `.status.ips` field). -### Network Interfaces +For Network Interfaces: * The annotation `test-network-function.com/defaultnetworkinterface` is the highest priority, and must contain a JSON-encoded string of the primary network interface for the pod. This must be explicitly set if needed. Examples can @@ -78,24 +63,15 @@ be seen in [cnf-certification-test-partner](https://github.com/test-network-func the first entry found with `"default"=true` is used. This annotation is automatically managed in OpenShift but may not be present in K8s. -### Container/pod based Specs +If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. `ping`), it should be +given the label `test-network-function.com/skip_connectivity_tests` to exclude it from those tests. The label value is +not important, only its presence. Equivalent to `excludeContainersFromConnectivityTests` in the config file. -* Pods to be tested by the `container` spec are identified with the `test-network-function.com/container=target` -label. Any value is permitted but `target` is used here for consistency with the `generic` spec. -* If specific tests are to be run on the pod then they can be listed as a JSON-encoded list of strings under the -`test-network-function.com/container_tests` annotation. If the annotation is not found then by default every -group of CNF tests defined in `testconfigure.yml` will be run. +If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, instrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. -For example: -```yaml -... - labels: - test-network-function.com/container: target - annotations: - test-network-function.com/container_tests: "[\"PRIVILEGED_POD\",\"PRIVILEGED_ROLE\"]" # optional -... -``` -### Operator Spec +#### operators + +The section can be configured as well as auto discovered. For manual configuration, see the commented part of the [sample config](test-network-function/tnf_config.yml). For auto discovery: * CSVs to be tested by the `operator` spec are identified with the `test-network-function-com/operator=target` label. Any value is permitted but `target` is used here for consistency with the other specs. @@ -104,15 +80,24 @@ annotation. This is equivalent to the `test-network-function.com/container_tests * `test-network-function.com/subscription_name` is optional and should contain a JSON-encoded string that's the name of the subscription for this CSV. If unset, the CSV name will be used. -### Runtime environement variables to skip tests during development -#### Turn off openshift required tests when CNF run on kubernetes only environment -when test on CNFs that run on k8s only environment, execute shell command below before compile tool and run test shell script. +### testPartner + +This section can also be discovered automatically and should be left commented out unless the parter pods are modified from the original version in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/local-test-infra/) + +### certifiedcontainerinfo and certifiedoperatorinfo + +The `certifiedcontainerinfo` and `certifiedoperatorinfo` sections contain information about CNFs and Operators that are +to be checked for certification status on Red Hat catalogs. + +## Runtime environement variables to skip or include tests +### Turn off openshift required tests +When test on CNFs that run on k8s only environment, execute shell command below before compile tool and run test shell script. -```shell-command +```shell script export TNF_MINIKUBE_ONLY=true ``` -#### Disable intrusive tests +### Disable intrusive tests If you would like to skip intrusive tests which may disrupt cluster operations, issue the following: ```shell script @@ -121,15 +106,15 @@ export TNF_NON_INTRUSIVE_ONLY=true Likewise, to enable intrusive tests, set the following: -```shell-command +```shell script export TNF_NON_INTRUSIVE_ONLY=false ``` -#### Execute test suites from openshift-kni/cnf-feature-deploy +### Execute test suites from openshift-kni/cnf-feature-deploy The test suites from openshift-kni/cnf-feature-deploy can be run prior to the actual CNF certification test execution and the results are incorporated in the same claim file if the following environment variable is set: -```shell-command +```shell script export VERIFY_CNF_FEATURES=true ``` @@ -163,7 +148,7 @@ Optional arguments are: If `-k` is not specified, autodiscovery is performed. The autodiscovery first looks for paths in the `$KUBECONFIG` environment variable on the host system, and if the variable is not set or is empty, the default configuration stored in `$HOME/.kube/config` is checked. -```shell-script +```shell script ./run-tnf-container.sh -k ~/.kube/config -t ~/tnf/config -o ~/tnf/output diagnostic access-control ``` @@ -172,7 +157,6 @@ The autodiscovery first looks for paths in the `$KUBECONFIG` environment variabl *Note*: The `run-tnf-container.sh` script performs autodiscovery of selected TNF environment variables. Currently supported environment variables include: - `TNF_MINIKUBE_ONLY` -- `TNF_ENABLE_CONFIG_AUTODISCOVER` ### Running using `docker` instead of `podman` @@ -189,13 +173,13 @@ export TNF_CONTAINER_CLIENT="docker" You can build an image locally by using the command below. Use the value of `TNF_VERSION` to set a branch, a tag, or a hash of a commit that will be installed into the image. -```shell-script +```shell script docker build -t test-network-function:v1.0.5 --build-arg TNF_VERSION=v1.0.5 . ``` To build an image that installs TNF from an unofficial source (e.g. a fork of the TNF repository), use the `TNF_SRC_URL` build argument to override the URL to a source repository. -```shell-script +```shell script docker build -t test-network-function:v1.0.5 \ --build-arg TNF_VERSION=v1.0.5 \ --build-arg TNF_SRC_URL=https://github.com/test-network-function/test-network-function . @@ -203,7 +187,7 @@ docker build -t test-network-function:v1.0.5 \ To make `run-tnf-container.sh` use the newly built image, specify the custom TNF image using the `-i` parameter. -```shell-script +```shell script ./run-tnf-container.sh -i test-network-function:v1.0.5 -t ~/tnf/config -o ~/tnf/output diagnostic access-control ``` Note: see [General tests](#general-tests) for a list of available keywords. @@ -234,13 +218,13 @@ Dependency|Minimum Version Other binary dependencies required to run tests can be installed using the following command: -```shell-script +```shell script make install-tools ``` Finally the source dependencies can be installed with -```shell-script +```shell script make update-deps ``` @@ -254,7 +238,7 @@ make update-deps In order to pull the code, issue the following command: -```shell-script +```shell script mkdir ~/workspace cd ~/workspace git clone git@github.com:test-network-function/test-network-function.git @@ -265,7 +249,7 @@ cd test-network-function In order to build the test executable, first make sure you have satisfied the [dependencies](#dependencies). -```shell-script +```shell script make build-cnf-tests ``` @@ -278,7 +262,7 @@ script. Run any combination of the suites keywords listed at in the [General tests](#general-tests) section, e.g. -```shell-script +```shell script ./run-cnf-suites.sh diagnostic ./run-cnf-suites.sh diagnostic lifecycle ./run-cnf-suites.sh diagnostic networking operator @@ -290,7 +274,7 @@ By default the claim file will be output into the same location as the test exec `run-cnf-suites.sh` can be used to provide a new location that the output files will be saved to. For more detailed control over the outputs, see the output of `test-network-function.test --help`. -```shell-script +```shell script cd test-network-function && ./test-network-function.test --help ``` @@ -528,7 +512,7 @@ TNF_DEFAULT_BUFFER_SIZE=32768 ./run-cnf-suites.sh diagnostic generic In some cases, containers do not provide ping or ip binary utilities. Since these binaries are required for the connectivity tests, we must exclude such containers from the connectivity test suite. In order to exclude these -containers, please issue add the following to `test-network-function/generic_test_configuration.yaml`: +containers, please add the following to `test-network-function/tnf_config.yml`: ```yaml excludeContainersFromConnectivityTests: diff --git a/docs/config.md b/docs/config.md deleted file mode 100644 index 6eb4ceeef..000000000 --- a/docs/config.md +++ /dev/null @@ -1,59 +0,0 @@ -## Test Configuration - -Configuration is accomplished with `tnf_config.yml` by default. An alternative configuration can be provided using the -`TNF_CONFIGURATION_PATH` environment variable. - -This config file contains several sections, each of which configures one or more test specs: - -Config Section|Purpose ----|--- -tagetPodLabels|A list of labels that will be used to discover the pods/containers under test -generic|Describes containers to be tested with the `generic` and `multus` specs, if they are run. -cnfs|Defines which containers are to be tested by the `container` spec. -operators|Defines which containers are to be tested by the `operator` spec. -certifiedcontainerinfo|Describes cnf names and repositories to be checked for certification status. -certifiedoperatorinfo|Describes operator names and organisations to be checked for certification status. - -`testconfigure.yml` defines roles, and which tests are appropriate for which roles. It should not be necessary to modify this. - -### targetPodLabels - -This section contains a list of labels that will be used to discover the pods/containers under test **in addition to** the ones -explicitly configured in the sections below or discovered through the default labels used by auto discovery. The containers -discovered in this way will be targeted for `generic`/`multus`/`container` specs if they are in focus. - -### generic - -The `generic` section contains three subsections: - -* `containersUnderTest:` describes the CNFs that will be tested. Each container is defined by the combination of its -`namespace`, `podName`, and `containerName`, which are also used to connect to the container when required. - - * Each entry for `containersUnderTest` must also define the `defaultNetworkDevice` of that container. There is also - an optional `multusIpAddresses` that can be omitted if the multus tests are not run. - -* `partnerContainers:` describes the containers that support the testing. Multiple `partnerContainers` allows -for more complex testing scenarios. At the time of writing, only one is used, which will also be the test -orchestrator. - -* `testOrchestrator:` references a partner containers that is used for the generic test suite. The test partner is used -to send various types of traffic to each container under test. For example the orchestrator is used to ping a container -under test, and to be the ping target of a container under test. - -The [included default](../test-network-function/tnf_config.yml) defines a single container to be tested, -and a single partner to do the testing. - -### cnfs and operators - -The `cnfs` and `operators` sections define the roles under which operators and containers are to be tested. - -[The default config](../test-network-function/tnf_config.yml) is set up with some examples of this: -It will run the `"OPERATOR_STATUS"` tests (as defined in `testconfigure.yml`) against an etcd operator, and the -`"PRIVILEGED_POD"` and `"PRIVILEGED_ROLE"` tests against an nginx container. - -A more extensive example of all these sections is provided in [example/example_config.yaml](../example/example_config.yaml) - -### certifiedcontainerinfo and certifiedoperatorinfo - -The `certifiedcontainerinfo` and `certifiedoperatorinfo` sections contain information about CNFs and Operators that are -to be checked for certification status on Red Hat catalogs. diff --git a/examples/example_config.yaml b/examples/example_config.yaml deleted file mode 100644 index dc10a21af..000000000 --- a/examples/example_config.yaml +++ /dev/null @@ -1,77 +0,0 @@ -targetPodLabels: - - namespace: example-cnf.com - name: example-cnf-deployment - value: example-app -generic: - containersUnderTest: - - namespace: tnf - podName: test - containerName: test - defaultNetworkDevice: eth0 - multusIpAddresses: - - 10.217.0.8 - partnerContainers: - - namespace: tnf - podName: partner - containerName: partner - defaultNetworkDevice: eth0 - multusIpAddresses: - - 10.217.0.29 - - namespace: tnf - podName: node-master - containerName: master - defaultNetworkDevice: eth0 - fsDiffMasterContainer: - namespace: tnf - podName: node-master - containerName: master - testOrchestrator: - namespace: tnf - podName: partner - containerName: partner -operators: - - name: etcdoperator.v0.9.4 - namespace: my-etcd - subscriptionName: etcd - status: Succeeded - autogenerate: "true" - crds: - - name: test.crd.one - namespace: default - instances: - - name: Instance_one - - name: test.crd.two - namespace: default - instances: - - name: Instance_two - deployments: - - name: deployment1 - replicas: "1" - permissions: - - name: name - role: clusterrole - cnfs: - - name: cnf_one - namespace: test - status: "" - tests: - - PRIVILEGED_POD - tests: - - CSV_INSTALLED - - SUBSCRIPTION_INSTALLED - - CSV_SCC -cnfs: - - name: cnf_only - namespace: test - status: "" - tests: - - PRIVILEGED_POD -certifiedcontainerinfo: - - name: nginx-116 - repository: rhel8 -certifiedoperatorinfo: - - name: etcd-operator - organization: redhat-marketplace -cnfavailabletestcases: - - PRIVILEGED_POD - - CLUSTER_ROLE diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index ca2b02c73..822d6736d 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -67,3 +67,63 @@ func makeGetCommand(resourceType, labelQuery string) *exec.Cmd { return cmd } + +// getContainersByLabel builds `config.Container`s from containers in pods matching a label. +func getContainersByLabel(label configsections.Label) (containers []configsections.Container, err error) { + pods, err := GetPodsByLabel(label) + if err != nil { + return nil, err + } + for i := range pods.Items { + containers = append(containers, buildContainersFromPodResource(&pods.Items[i])...) + } + return containers, nil +} + +// getContainerIdentifiersByLabel builds `config.ContainerIdentifier`s from containers in pods matching a label. +func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs []configsections.ContainerIdentifier, err error) { + containers, err := getContainersByLabel(label) + if err != nil { + return nil, err + } + for _, c := range containers { + containerIDs = append(containerIDs, c.ContainerIdentifier) + } + return containerIDs, nil +} + +// getContainerByLabel returns exactly one container with the given label. If any other number of containers is found +// then an error is returned along with an empty `config.Container`. +func getContainerByLabel(label configsections.Label) (container configsections.Container, err error) { + containers, err := getContainersByLabel(label) + if err != nil { + return container, err + } + if len(containers) != 1 { + return container, fmt.Errorf("expected exactly one container, got %d for label %s/%s=%s", len(containers), label.Namespace, label.Name, label.Value) + } + return containers[0], nil +} + +// buildContainersFromPodResource builds `configsections.Container`s from a `PodResource` +func buildContainersFromPodResource(pr *PodResource) (containers []configsections.Container) { + for _, containerResource := range pr.Spec.Containers { + var err error + var container configsections.Container + container.Namespace = pr.Metadata.Namespace + container.PodName = pr.Metadata.Name + container.ContainerName = containerResource.Name + container.DefaultNetworkDevice, err = pr.getDefaultNetworkDeviceFromAnnotations() + if err != nil { + log.Warnf("error encountered getting default network device: %s", err) + } + container.MultusIPAddresses, err = pr.getPodIPs() + if err != nil { + log.Warnf("error encountered getting multus IPs: %s", err) + err = nil + } + + containers = append(containers, container) + } + return +} diff --git a/pkg/config/autodiscover/autodiscover_cnfs.go b/pkg/config/autodiscover/autodiscover_cnfs.go deleted file mode 100644 index 71f8982a1..000000000 --- a/pkg/config/autodiscover/autodiscover_cnfs.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) 2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package autodiscover - -import ( - log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/config/configsections" - "github.com/test-network-function/test-network-function/pkg/tnf/testcases" -) - -const ( - cnfLabelName = "container" - configuredTestFile = "testconfigure.yml" -) - -var ( - cnfTestsAnnotationName = buildAnnotationName("container_tests") -) - -// BuildCNFsConfig builds a `[]configsections.Cnf` from the current state of the cluster, -// using labels and annotations to populate the data. -func BuildCNFsConfig() (cnfs []configsections.Cnf) { - pods, err := GetPodsByLabel(configsections.Label{Namespace: tnfNamespace, Name: cnfLabelName, Value: anyLabelValue}) - if err != nil { - log.Fatalf("found no CNFs to test while 'container' spec enabled: %s", err) - } - for i := range pods.Items { - cnfs = append(cnfs, BuildCnfFromPodResource(&pods.Items[i])) - } - return cnfs -} - -// BuildCnfFromPodResource builds a single `configsections.Cnf` from a PodResource -func BuildCnfFromPodResource(pr *PodResource) (cnf configsections.Cnf) { - var err error - cnf.Namespace = pr.Metadata.Namespace - cnf.Name = pr.Metadata.Name - - var tests []string - err = pr.GetAnnotationValue(cnfTestsAnnotationName, &tests) - if err != nil { - log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", cnf.Namespace, cnf.Name, err) - cnf.Tests = getConfiguredCNFTests() - } else { - cnf.Tests = tests - } - return -} - -// getConfiguredCNFTests loads the `configuredTestFile` used by the `operator` and `container` specs, and extracts -// the names of test groups from it. -func getConfiguredCNFTests() (cnfTests []string) { - configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) - if err != nil { - log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) - return []string{} - } - for _, configuredTest := range configuredTests.CnfTest { - cnfTests = append(cnfTests, configuredTest.Name) - } - log.WithField("cnfTests", cnfTests).Infof("got all tests from %s.", configuredTestFile) - return cnfTests -} diff --git a/pkg/config/autodiscover/autodiscover_generic.go b/pkg/config/autodiscover/autodiscover_generic.go deleted file mode 100644 index fdb1c1ef0..000000000 --- a/pkg/config/autodiscover/autodiscover_generic.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package autodiscover - -import ( - "fmt" - - log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/config/configsections" -) - -const ( - genericLabelName = "generic" - underTestValue = "target" - // partnerLabelValue = "partner" - orchestratorValue = "orchestrator" - fsDiffMasterValue = "fs_diff_master" - skipConnectivityTestsLabel = "skip_connectivity_tests" -) - -// BuildGenericConfig builds a `configsections.TestConfiguration` from the current state of the cluster, -// using labels and annotations to populate the data. -func BuildGenericConfig() (conf configsections.TestConfiguration) { - var partnerContainers []configsections.Container // PartnerContainers is built from all non-target containers - - // an orchestrator must be identified - orchestrator, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: orchestratorValue}) - if err != nil { - log.Fatalf("failed to identify a single test orchestrator container: %s", err) - } - partnerContainers = append(partnerContainers, orchestrator) - conf.TestOrchestrator = orchestrator.ContainerIdentifier - - // there must be containers to test - containersUnderTest, err := GetContainersByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: underTestValue}) - if err != nil { - log.Fatalf("found no containers to test: %s", err) - } - conf.ContainersUnderTest = containersUnderTest - - // the FS Diff master container is optional - fsDiffMasterContainer, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: fsDiffMasterValue}) - if err == nil { - partnerContainers = append(partnerContainers, fsDiffMasterContainer) - conf.FsDiffMasterContainer = fsDiffMasterContainer.ContainerIdentifier - } else { - log.Warnf("an error (%s) occurred when getting the FS Diff Master Container. Attempting to continue", err) - } - - // Containers to exclude from connectivity tests are optional - connectivityExcludedContainers, err := getContainerIdentifiersByLabel(configsections.Label{Namespace: tnfNamespace, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) - if err != nil { - log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) - } - conf.ExcludeContainersFromConnectivityTests = connectivityExcludedContainers - - conf.PartnerContainers = partnerContainers - - return conf -} - -// GetContainersByLabel builds `config.Container`s from containers in pods matching a label. -func GetContainersByLabel(label configsections.Label) (containers []configsections.Container, err error) { - pods, err := GetPodsByLabel(label) - if err != nil { - return nil, err - } - for i := range pods.Items { - containers = append(containers, BuildContainersFromPodResource(&pods.Items[i])...) - } - return containers, nil -} - -// getContainerIdentifiersByLabel builds `config.ContainerIdentifier`s from containers in pods matching a label. -func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs []configsections.ContainerIdentifier, err error) { - containers, err := GetContainersByLabel(label) - if err != nil { - return nil, err - } - for _, c := range containers { - containerIDs = append(containerIDs, c.ContainerIdentifier) - } - return containerIDs, nil -} - -// getContainerByLabel returns exactly one container with the given label. If any other number of containers is found -// then an error is returned along with an empty `config.Container`. -func getContainerByLabel(label configsections.Label) (container configsections.Container, err error) { - containers, err := GetContainersByLabel(label) - if err != nil { - return container, err - } - if len(containers) != 1 { - return container, fmt.Errorf("expected exactly one container, got %d for label %s/%s=%s", len(containers), label.Namespace, label.Name, label.Value) - } - return containers[0], nil -} - -// BuildContainersFromPodResource builds `configsections.Container`s from a `PodResource` -func BuildContainersFromPodResource(pr *PodResource) (containers []configsections.Container) { - for _, containerResource := range pr.Spec.Containers { - var err error - var container configsections.Container - container.Namespace = pr.Metadata.Namespace - container.PodName = pr.Metadata.Name - container.ContainerName = containerResource.Name - container.DefaultNetworkDevice, err = pr.getDefaultNetworkDeviceFromAnnotations() - if err != nil { - log.Warnf("error encountered getting default network device: %s", err) - } - container.MultusIPAddresses, err = pr.getPodIPs() - if err != nil { - log.Warnf("error encountered getting multus IPs: %s", err) - err = nil - } - - containers = append(containers, container) - } - return -} diff --git a/pkg/config/autodiscover/autodiscover_operator.go b/pkg/config/autodiscover/autodiscover_operator.go deleted file mode 100644 index 573da1e08..000000000 --- a/pkg/config/autodiscover/autodiscover_operator.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package autodiscover - -import ( - log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/config/configsections" - "github.com/test-network-function/test-network-function/pkg/tnf/testcases" -) - -const ( - operatorLabelName = "operator" -) - -var ( - operatorTestsAnnotationName = buildAnnotationName("operator_tests") - subscriptionNameAnnotationName = buildAnnotationName("subscription_name") -) - -// BuildOperatorConfig builds a `[]configsections.Operator` from the current state of the cluster, -// using labels and annotations to populate the data. -func BuildOperatorConfig() (operatorsToTest []configsections.Operator) { - csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue) - if err != nil { - log.Fatalf("found no CSVs to test while 'operator' spec enabled: %s", err) - } - for i := range csvs.Items { - operatorsToTest = append(operatorsToTest, BuildOperatorFromCSVResource(&csvs.Items[i])) - } - return operatorsToTest -} - -// BuildOperatorFromCSVResource builds a single `configsections.Operator` from a CSVResource -func BuildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) { - var err error - op.Name = csv.Metadata.Name - op.Namespace = csv.Metadata.Namespace - - var tests []string - err = csv.GetAnnotationValue(operatorTestsAnnotationName, &tests) - if err != nil { - log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", op.Namespace, op.Name, err) - op.Tests = getConfiguredOperatorTests() - } else { - op.Tests = tests - } - - var subscriptionName string - err = csv.GetAnnotationValue(subscriptionNameAnnotationName, &subscriptionName) - if err != nil { - log.Warnf("unable to get a subscription name annotation from CSV %s (%s), the CSV name will be used", csv.Metadata.Name, err) - } - op.SubscriptionName = subscriptionName - - return op -} - -// getConfiguredOperatorTests loads the `configuredTestFile` used by the `operator` specs and extracts -// the names of test groups from it. -func getConfiguredOperatorTests() (opTests []string) { - configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) - if err != nil { - log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) - return []string{} - } - for _, configuredTest := range configuredTests.OperatorTest { - opTests = append(opTests, configuredTest.Name) - } - log.WithField("opTests", opTests).Infof("got all tests from %s.", configuredTestFile) - return opTests -} diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go new file mode 100644 index 000000000..3b8343dfb --- /dev/null +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -0,0 +1,51 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/config/configsections" +) + +const ( + genericLabelName = "generic" + orchestratorValue = "orchestrator" + fsDiffMasterValue = "fs_diff_master" +) + +// FillTestPartner completes a `configsections.TestPartner` from the current state of the cluster, +// using labels and annotations to populate the data, if it's not fully configured +func FillTestPartner(tp *configsections.TestPartner) { + if tp.TestOrchestrator.ContainerName == "" { + orchestrator, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: orchestratorValue}) + if err != nil { + log.Fatalf("failed to identify a single test orchestrator container: %s", err) + } + tp.PartnerContainers = append(tp.PartnerContainers, orchestrator) + tp.TestOrchestrator = orchestrator.ContainerIdentifier + } + + if tp.FsDiffMasterContainer.ContainerName == "" { + fsDiffMasterContainer, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: fsDiffMasterValue}) + if err == nil { + tp.PartnerContainers = append(tp.PartnerContainers, fsDiffMasterContainer) + tp.FsDiffMasterContainer = fsDiffMasterContainer.ContainerIdentifier + } else { + log.Warnf("an error (%s) occurred when getting the FS Diff Master Container. Attempting to continue", err) + } + } +} diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go new file mode 100644 index 000000000..52700ef7f --- /dev/null +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -0,0 +1,142 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf/testcases" +) + +const ( + configuredTestFile = "testconfigure.yml" + operatorLabelName = "operator" + skipConnectivityTestsLabel = "skip_connectivity_tests" +) + +var ( + operatorTestsAnnotationName = buildAnnotationName("operator_tests") + subscriptionNameAnnotationName = buildAnnotationName("subscription_name") + podTestsAnnotationName = buildAnnotationName("host_resource_tests") +) + +// FindTestTarget builds a `configsections.TestTarget` from the current state of the cluster, +// using labels and annotations to populate the data. +func FindTestTarget(labels []configsections.Label) (target configsections.TestTarget) { + // find pods by label + for _, l := range labels { + pods, err := GetPodsByLabel(l) + if err == nil { + for i := range pods.Items { + target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(&pods.Items[i])) + target.ContainersUnderTest = append(target.ContainersUnderTest, buildContainersFromPodResource(&pods.Items[i])...) + } + } else { + log.Warnf("failed to query by label: %v %v", l, err) + } + } + // Containers to exclude from connectivity tests are optional + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Namespace: tnfNamespace, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) + target.ExcludeContainersFromConnectivityTests = identifiers + + if err != nil { + log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) + } + + csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue) + if err == nil { + for i := range csvs.Items { + target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csvs.Items[i])) + } + } else { + log.Warnf("an error (%s) occurred when looking for operaters by label", err) + } + + return target +} + +// buildPodUnderTest builds a single `configsections.Pod` from a PodResource +func buildPodUnderTest(pr *PodResource) (cnf configsections.Pod) { + var err error + cnf.Namespace = pr.Metadata.Namespace + cnf.Name = pr.Metadata.Name + + var tests []string + err = pr.GetAnnotationValue(podTestsAnnotationName, &tests) + if err != nil { + log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", cnf.Namespace, cnf.Name, err) + cnf.Tests = getConfiguredPodTests() + } else { + cnf.Tests = tests + } + return +} + +// getConfiguredPodTests loads the `configuredTestFile` and extracts +// the names of test groups from it. +func getConfiguredPodTests() (cnfTests []string) { + configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) + if err != nil { + log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) + return []string{} + } + for _, configuredTest := range configuredTests.CnfTest { + cnfTests = append(cnfTests, configuredTest.Name) + } + log.WithField("cnfTests", cnfTests).Infof("got all tests from %s.", configuredTestFile) + return cnfTests +} + +// buildOperatorFromCSVResource builds a single `configsections.Operator` from a CSVResource +func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) { + var err error + op.Name = csv.Metadata.Name + op.Namespace = csv.Metadata.Namespace + + var tests []string + err = csv.GetAnnotationValue(operatorTestsAnnotationName, &tests) + if err != nil { + log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", op.Namespace, op.Name, err) + op.Tests = getConfiguredOperatorTests() + } else { + op.Tests = tests + } + + var subscriptionName string + err = csv.GetAnnotationValue(subscriptionNameAnnotationName, &subscriptionName) + if err != nil { + log.Warnf("unable to get a subscription name annotation from CSV %s (%s), the CSV name will be used", csv.Metadata.Name, err) + } + op.SubscriptionName = subscriptionName + + return op +} + +// getConfiguredOperatorTests loads the `configuredTestFile` used by the `operator` specs and extracts +// the names of test groups from it. +func getConfiguredOperatorTests() (opTests []string) { + configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) + if err != nil { + log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) + return []string{} + } + for _, configuredTest := range configuredTests.OperatorTest { + opTests = append(opTests, configuredTest.Name) + } + log.WithField("opTests", opTests).Infof("got all tests from %s.", configuredTestFile) + return opTests +} diff --git a/pkg/config/autodiscover/container_test.go b/pkg/config/autodiscover/container_test.go index d443a9455..977ed305c 100644 --- a/pkg/config/autodiscover/container_test.go +++ b/pkg/config/autodiscover/container_test.go @@ -14,29 +14,38 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package autodiscover_test +package autodiscover import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config/autodiscover" ) -func TestBuildCnfFromPodResource(t *testing.T) { - orchestratorPodResource := loadPodResource(testOrchestratorFilePath) - orchestratorPod := autodiscover.BuildCnfFromPodResource(&orchestratorPodResource) +func TestBuildContainersFromPodResource(t *testing.T) { + orchestratorPod := loadPodResource(testOrchestratorFilePath) + orchestratorContainers := buildContainersFromPodResource(&orchestratorPod) + assert.Equal(t, 1, len(orchestratorContainers)) - subjectPodResource := loadPodResource(testSubjectFilePath) - subjectPod := autodiscover.BuildCnfFromPodResource(&subjectPodResource) + subjectPod := loadPodResource(testSubjectFilePath) + subjectContainers := buildContainersFromPodResource(&subjectPod) + assert.Equal(t, 1, len(subjectContainers)) - assert.Equal(t, "tnf", orchestratorPod.Namespace) - assert.Equal(t, "I'mAPodName", orchestratorPod.Name) - assert.NotEqual(t, "I'mAContainer", orchestratorPod.Name) - // no tests set on pod and the config file will not be loaded from the unit test context: no tests should be set. - assert.Equal(t, []string{}, orchestratorPod.Tests) + assert.Equal(t, "tnf", orchestratorContainers[0].Namespace) + assert.Equal(t, "I'mAPodName", orchestratorContainers[0].PodName) + assert.Equal(t, "I'mAContainer", orchestratorContainers[0].ContainerName) - assert.Equal(t, "tnf", subjectPod.Namespace) - assert.Equal(t, "test", subjectPod.Name) - assert.Equal(t, []string{"OneTestName", "AnotherTestName"}, subjectPod.Tests) + // Check correct order of precedence for network devices + assert.Equal(t, "eth0", orchestratorContainers[0].DefaultNetworkDevice) + assert.NotEqual(t, "LowerPriorityInterface", orchestratorContainers[0].DefaultNetworkDevice) + assert.Equal(t, "eth1", subjectContainers[0].DefaultNetworkDevice) + + // Check correct IPs are chosen + assert.Equal(t, 1, len(orchestratorContainers[0].MultusIPAddresses)) + assert.Equal(t, "1.1.1.1", orchestratorContainers[0].MultusIPAddresses[0]) + assert.NotEqual(t, "2.2.2.2", orchestratorContainers[0].MultusIPAddresses[0]) + // test-network-function.com/multusips should be used for the test subject container. + assert.Equal(t, 2, len(subjectContainers[0].MultusIPAddresses)) + assert.Equal(t, "3.3.3.3", subjectContainers[0].MultusIPAddresses[0]) + assert.Equal(t, "4.4.4.4", subjectContainers[0].MultusIPAddresses[1]) } diff --git a/pkg/config/autodiscover/csv_info_test.go b/pkg/config/autodiscover/csv_info_test.go index 0ac491260..3011a806e 100644 --- a/pkg/config/autodiscover/csv_info_test.go +++ b/pkg/config/autodiscover/csv_info_test.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package autodiscover_test +package autodiscover import ( "encoding/json" @@ -24,7 +24,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config/autodiscover" ) const ( @@ -35,7 +34,7 @@ var ( csvFilePath = path.Join(filePath, csvFile) ) -func loadCSVResource(filePath string) (csv autodiscover.CSVResource) { +func loadCSVResource(filePath string) (csv CSVResource) { contents, err := ioutil.ReadFile(filePath) if err != nil { log.Fatalf("error (%s) loading CSVResource %s for testing", err, filePath) diff --git a/pkg/config/autodiscover/generic_test.go b/pkg/config/autodiscover/generic_test.go deleted file mode 100644 index 568ace234..000000000 --- a/pkg/config/autodiscover/generic_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package autodiscover_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config/autodiscover" -) - -func TestBuildContainersFromPodResource(t *testing.T) { - orchestratorPod := loadPodResource(testOrchestratorFilePath) - orchestratorContainers := autodiscover.BuildContainersFromPodResource(&orchestratorPod) - assert.Equal(t, 1, len(orchestratorContainers)) - - subjectPod := loadPodResource(testSubjectFilePath) - subjectContainers := autodiscover.BuildContainersFromPodResource(&subjectPod) - assert.Equal(t, 1, len(subjectContainers)) - - assert.Equal(t, "tnf", orchestratorContainers[0].Namespace) - assert.Equal(t, "I'mAPodName", orchestratorContainers[0].PodName) - assert.Equal(t, "I'mAContainer", orchestratorContainers[0].ContainerName) - - // Check correct order of precedence for network devices - assert.Equal(t, "eth0", orchestratorContainers[0].DefaultNetworkDevice) - assert.NotEqual(t, "LowerPriorityInterface", orchestratorContainers[0].DefaultNetworkDevice) - assert.Equal(t, "eth1", subjectContainers[0].DefaultNetworkDevice) - - // Check correct IPs are chosen - assert.Equal(t, 1, len(orchestratorContainers[0].MultusIPAddresses)) - assert.Equal(t, "1.1.1.1", orchestratorContainers[0].MultusIPAddresses[0]) - assert.NotEqual(t, "2.2.2.2", orchestratorContainers[0].MultusIPAddresses[0]) - // test-network-function.com/multusips should be used for the test subject container. - assert.Equal(t, 2, len(subjectContainers[0].MultusIPAddresses)) - assert.Equal(t, "3.3.3.3", subjectContainers[0].MultusIPAddresses[0]) - assert.Equal(t, "4.4.4.4", subjectContainers[0].MultusIPAddresses[1]) -} diff --git a/pkg/config/autodiscover/operator_test.go b/pkg/config/autodiscover/operator_test.go index 166a961fd..21e4b3191 100644 --- a/pkg/config/autodiscover/operator_test.go +++ b/pkg/config/autodiscover/operator_test.go @@ -14,18 +14,17 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package autodiscover_test +package autodiscover import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config/autodiscover" ) func TestBuildOperatorFromCSVResource(t *testing.T) { csvResource := loadCSVResource(csvFilePath) - operator := autodiscover.BuildOperatorFromCSVResource(&csvResource) + operator := buildOperatorFromCSVResource(&csvResource) assert.Equal(t, "CSVNamespace", operator.Namespace) assert.Equal(t, "CSVName", operator.Name) diff --git a/pkg/config/autodiscover/pod_info_test.go b/pkg/config/autodiscover/pod_info_test.go index e3d43a865..e8c46d6b0 100644 --- a/pkg/config/autodiscover/pod_info_test.go +++ b/pkg/config/autodiscover/pod_info_test.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package autodiscover_test +package autodiscover import ( "encoding/json" @@ -24,7 +24,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config/autodiscover" ) const ( @@ -38,7 +37,7 @@ var ( testSubjectFilePath = path.Join(filePath, testSubjectFile) ) -func loadPodResource(filePath string) (pod autodiscover.PodResource) { +func loadPodResource(filePath string) (pod PodResource) { contents, err := ioutil.ReadFile(filePath) if err != nil { log.Fatalf("error (%s) loading PodResource %s for testing", err, filePath) diff --git a/pkg/config/autodiscover/pod_test.go b/pkg/config/autodiscover/pod_test.go new file mode 100644 index 000000000..2e3b95514 --- /dev/null +++ b/pkg/config/autodiscover/pod_test.go @@ -0,0 +1,41 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPodUnderTest(t *testing.T) { + orchestratorPodResource := loadPodResource(testOrchestratorFilePath) + orchestratorPod := buildPodUnderTest(&orchestratorPodResource) + + subjectPodResource := loadPodResource(testSubjectFilePath) + subjectPod := buildPodUnderTest(&subjectPodResource) + + assert.Equal(t, "tnf", orchestratorPod.Namespace) + assert.Equal(t, "I'mAPodName", orchestratorPod.Name) + assert.NotEqual(t, "I'mAContainer", orchestratorPod.Name) + // no tests set on pod and the config file will not be loaded from the unit test context: no tests should be set. + assert.Equal(t, []string{}, orchestratorPod.Tests) + + assert.Equal(t, "tnf", subjectPod.Namespace) + assert.Equal(t, "test", subjectPod.Name) + assert.Equal(t, []string{"OneTestName", "AnotherTestName"}, subjectPod.Tests) +} diff --git a/pkg/config/autodiscover/testdata/testtarget.json b/pkg/config/autodiscover/testdata/testtarget.json index 274d34b53..49cb3302c 100644 --- a/pkg/config/autodiscover/testdata/testtarget.json +++ b/pkg/config/autodiscover/testdata/testtarget.json @@ -4,7 +4,7 @@ "annotations": { "k8s.v1.cni.cncf.io/networks-status": "[{\n \"name\": \"\",\n \"interface\": \"eth1\",\n \"ips\": [\n \"10.217.1.89\"\n ],\n \"default\": true,\n \"dns\": {}\n}]", "test-network-function.com/multusips": "[\"3.3.3.3\",\"4.4.4.4\"]", - "test-network-function.com/container_tests": "[\"OneTestName\",\"AnotherTestName\"]" + "test-network-function.com/host_resource_tests": "[\"OneTestName\",\"AnotherTestName\"]" }, "labels": { "app": "test", diff --git a/pkg/config/config.go b/pkg/config/config.go index 0dd616370..fdf81e810 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,11 +21,9 @@ import ( "io/ioutil" "os" - ginkgoconfig "github.com/onsi/ginkgo/config" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/autodiscover" "github.com/test-network-function/test-network-function/pkg/config/configsections" - "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "gopkg.in/yaml.v2" ) @@ -34,37 +32,9 @@ const ( defaultConfigurationFilePath = "tnf_config.yml" ) -const ( - containerTestSpecName = "access-control" - operatorTestSpecName = "operator" -) - -// File is the top level of the config file. All new config sections must be added here -type File struct { - // Custom Pod labels for discovering containers under test for generic and container suites - TargetPodLabels []configsections.Label `yaml:"targetPodLabels,omitempty" json:"targetPodLabels,omitempty"` - - Generic configsections.TestConfiguration `yaml:"generic,omitempty" json:"generic,omitempty"` - - // Operator is the list of operator objects that needs to be tested. - Operators []configsections.Operator `yaml:"operators,omitempty" json:"operators,omitempty"` - - // CNFs is the list of the CNFs that needs to be tested. Each entry is a single pod to be tested. - CNFs []configsections.Cnf `yaml:"cnfs,omitempty" json:"cnfs,omitempty"` - - // CertifiedContainerInfo is the list of container images to be checked for certification status. - CertifiedContainerInfo []configsections.CertifiedContainerRequestInfo `yaml:"certifiedcontainerinfo,omitempty" json:"certifiedcontainerinfo,omitempty"` - - // CertifiedOperatorInfo is list of operator bundle names that are queried for certification status. - CertifiedOperatorInfo []configsections.CertifiedOperatorRequestInfo `yaml:"certifiedoperatorinfo,omitempty" json:"certifiedoperatorinfo,omitempty"` - - // CnfAvailableTestCases list the available test cases for reference. - CnfAvailableTestCases []string `yaml:"cnfavailabletestcases,omitempty" json:"cnfavailabletestcases,omitempty"` -} - var ( // configInstance is the singleton instance of loaded config, accessed through GetConfigInstance - configInstance File + configInstance configsections.TestConfiguration // loaded tracks if the config has been loaded to prevent it being reloaded. loaded = false // set when an intrusive test has done something that would cause Pod/Container to be recreated @@ -100,22 +70,8 @@ func loadConfigFromFile(filePath string) error { return nil } -// doAutodiscovery will autodiscover config for any enabled test spec. Specs which are not selected will be skipped to -// avoid unnecessary noise in the logs. -func doAutodiscovery() { - if genericTestConfigRequired() { - configInstance.Generic = autodiscover.BuildGenericConfig() - } - if podTestConfigRequired() { - configInstance.CNFs = autodiscover.BuildCNFsConfig() - } - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, operatorTestSpecName) { - configInstance.Operators = autodiscover.BuildOperatorConfig() - } -} - // GetConfigInstance provides access to the singleton ConfigFile instance. -func GetConfigInstance() File { +func GetConfigInstance() configsections.TestConfiguration { if !loaded { filePath := getConfigurationFilePathFromEnvironment() log.Debugf("GetConfigInstance before config loaded, loading from file: %s", filePath) @@ -123,65 +79,21 @@ func GetConfigInstance() File { if err != nil { log.Fatalf("unable to load configuration file: %s", err) } - - BuildConfig() + autodiscover.FillTestPartner(&configInstance.TestPartner) + discoverTestTargets() } else if needsRefresh { - BuildConfig() + discoverTestTargets() } return configInstance } -func findContainersByLabels(labels []configsections.Label) (containers []configsections.Container) { - for _, l := range labels { - list, err := autodiscover.GetContainersByLabel(l) - if err == nil { - containers = append(containers, list...) - } else { - log.Warnf("failed to query by label: %v %v", l, err) - } - } - return containers -} - -func findPodsByLabels(labels []configsections.Label) (cnfs []configsections.Cnf) { - for _, l := range labels { - pods, err := autodiscover.GetPodsByLabel(l) - if err == nil { - for i := range pods.Items { - cnfs = append(cnfs, autodiscover.BuildCnfFromPodResource(&pods.Items[i])) - } - } else { - log.Warnf("failed to query by label: %v %v", l, err) - } - } - return cnfs -} - -// BuildConfig does auto discovery based on default labels if enabled and additional target pod/container -// discovery based on custom labels -func BuildConfig() { +func discoverTestTargets() { if autodiscover.PerformAutoDiscovery() { - log.Warn("doing configuration autodiscovery. Currently this WILL override parts of the configuration file") - doAutodiscovery() - } - if genericTestConfigRequired() { - configInstance.Generic.ContainersUnderTest = append(configInstance.Generic.ContainersUnderTest, findContainersByLabels(configInstance.TargetPodLabels)...) - } - if podTestConfigRequired() { - configInstance.CNFs = append(configInstance.CNFs, findPodsByLabels(configInstance.TargetPodLabels)...) + configInstance.TestTarget = autodiscover.FindTestTarget(configInstance.TargetPodLabels) } needsRefresh = false } -func genericTestConfigRequired() bool { - // TODO clean up as part of config api refactoring task - return true -} - -func podTestConfigRequired() bool { - return testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, containerTestSpecName) -} - // SetNeedsRefresh marks the config stale so that the next getInstance call will redo discovery func SetNeedsRefresh() { needsRefresh = true diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ffacf749c..0ed37790a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,9 +30,9 @@ func TestLoadConfigFromFile(t *testing.T) { assert.Nil(t, loadConfigFromFile(filePath)) assert.NotNil(t, loadConfigFromFile(filePath)) // Loading when already loaded is an error case conf := GetConfigInstance() - assert.Equal(t, conf.Generic.TestOrchestrator.Namespace, "default") - assert.Equal(t, conf.Generic.TestOrchestrator.ContainerName, "partner") - assert.Equal(t, conf.Generic.TestOrchestrator.PodName, "partner") + assert.Equal(t, conf.TestOrchestrator.Namespace, "default") + assert.Equal(t, conf.TestOrchestrator.ContainerName, "partner") + assert.Equal(t, conf.TestOrchestrator.PodName, "partner") } func TestGetConfigInstance(t *testing.T) { diff --git a/pkg/config/configsections/certified_request_test.go b/pkg/config/configsections/certified_request_test.go index c54bd9c0c..6559c5a3a 100644 --- a/pkg/config/configsections/certified_request_test.go +++ b/pkg/config/configsections/certified_request_test.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package configsections_test +package configsections import ( "encoding/json" @@ -24,8 +24,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config" - "github.com/test-network-function/test-network-function/pkg/config/configsections" "gopkg.in/yaml.v2" ) @@ -40,22 +38,22 @@ type unmarshalFunc func([]byte, interface{}) error // test data var ( // bananas go in the fruit bowl. - fruitbowlRequestInfo = configsections.CertifiedContainerRequestInfo{ + fruitbowlRequestInfo = CertifiedContainerRequestInfo{ Name: "banana", Repository: "fruitbowl", } // apples go in the fridge. - fridgeRequestInfo = configsections.CertifiedContainerRequestInfo{ + fridgeRequestInfo = CertifiedContainerRequestInfo{ Name: "apple", Repository: "fridge", } - jenkinsOperatorRequestInfo = configsections.CertifiedOperatorRequestInfo{ + jenkinsOperatorRequestInfo = CertifiedOperatorRequestInfo{ Name: "jenkins", Organization: "Red Hat", } - etcdOperatorRequestInfo = configsections.CertifiedOperatorRequestInfo{ + etcdOperatorRequestInfo = CertifiedOperatorRequestInfo{ Name: "etcd", Organization: "Core OS", } @@ -78,14 +76,14 @@ func setupRequestTest(marshalFun marshalFunc) (tempfileName string) { return tempfile.Name() } -// loadRequestConfig reads `tmpPath`, unmarshals it using `unmarshalFun`, and returns the resulting `config.File`. -func loadRequestConfig(tmpPath string, unmarshalFun unmarshalFunc) (conf *config.File) { +// loadRequestConfig reads `tmpPath`, unmarshals it using `unmarshalFun`, and returns the resulting `TestConfiguration`. +func loadRequestConfig(tmpPath string, unmarshalFun unmarshalFunc) (conf *TestConfiguration) { contents, err := ioutil.ReadFile(tmpPath) if err != nil { log.Fatal(err) } - conf = &config.File{} + conf = &TestConfiguration{} err = unmarshalFun(contents, conf) if err != nil { log.Fatal(err) @@ -95,7 +93,7 @@ func loadRequestConfig(tmpPath string, unmarshalFun unmarshalFunc) (conf *config } // saveRequestConfig calls `marshalFun` on `c`, then writes the result to `configPath`. -func saveRequestConfig(marshalFun marshalFunc, c *config.File, configPath string) { +func saveRequestConfig(marshalFun marshalFunc, c *TestConfiguration, configPath string) { bytes, err := marshalFun(c) if err != nil { log.Fatal(err) @@ -113,13 +111,13 @@ func cleanupTempfiles() { tempFiles = make([]*os.File, 0) } -func buildRequestConfig() *config.File { - conf := &config.File{} - conf.CertifiedContainerInfo = []configsections.CertifiedContainerRequestInfo{ +func buildRequestConfig() *TestConfiguration { + conf := &TestConfiguration{} + conf.CertifiedContainerInfo = []CertifiedContainerRequestInfo{ fruitbowlRequestInfo, fridgeRequestInfo, } - conf.CertifiedOperatorInfo = []configsections.CertifiedOperatorRequestInfo{ + conf.CertifiedOperatorInfo = []CertifiedOperatorRequestInfo{ jenkinsOperatorRequestInfo, etcdOperatorRequestInfo, } diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 16dc2fcd3..0a4d7002d 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -22,3 +22,56 @@ type Label struct { Name string `yaml:"name" json:"name"` Value string `yaml:"value" json:"value"` } + +// Operator struct defines operator manifest for testing +type Operator struct { + + // Name is a required field, Name of the csv . + Name string `yaml:"name" json:"name"` + + // Namespace is a required field , namespace is where the csv is installed. + // If its all namespace then you can replace it with ALL_NAMESPACE TODO: add check for ALL_NAMESPACE + Namespace string `yaml:"namespace" json:"namespace"` + + // Tests this is list of test that need to run against the operator. + Tests []string `yaml:"tests" json:"tests"` + + // Subscription name is required field, Name of used subscription. + SubscriptionName string `yaml:"subscriptionName" json:"subscriptionName"` +} + +// TestConfiguration provides test related configuration +type TestConfiguration struct { + // Custom Pod labels for discovering containers/pods under test + TargetPodLabels []Label `yaml:"targetPodLabels,omitempty" json:"targetPodLabels,omitempty"` + // TestTarget contains k8s resources that can be targeted by tests + TestTarget `yaml:"testTarget" json:"testTarget"` + // TestPartner contains the helper containers that can be used to facilitate tests + TestPartner `yaml:"testPartner" json:"testPartner"` + // CertifiedContainerInfo is the list of container images to be checked for certification status. + CertifiedContainerInfo []CertifiedContainerRequestInfo `yaml:"certifiedcontainerinfo,omitempty" json:"certifiedcontainerinfo,omitempty"` + // CertifiedOperatorInfo is list of operator bundle names that are queried for certification status. + CertifiedOperatorInfo []CertifiedOperatorRequestInfo `yaml:"certifiedoperatorinfo,omitempty" json:"certifiedoperatorinfo,omitempty"` +} + +// TestPartner contains the helper containers that can be used to facilitate tests +type TestPartner struct { + // PartnerContainers is the list parter containers that facilitates tests + PartnerContainers []Container `yaml:"partnerContainers" json:"partnerContainers"` + // TestOrchestrator is the id of the partner container for conducting connectivity tests + TestOrchestrator ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` + // FsDiffMasterContainer is the id of the partner container for conducting base image comparison + FsDiffMasterContainer ContainerIdentifier `yaml:"fsDiffMasterContainer" json:"fsDiffMasterContainer"` +} + +// TestTarget is a collection of resources under test +type TestTarget struct { + // PodsUnderTest is the list of the pods that needs to be tested. Each entry is a single pod to be tested. + PodsUnderTest []Pod `yaml:"podsUnderTest,omitempty" json:"podsUnderTest,omitempty"` + // ContainersUnderTest is the list of containers that needs to be tested. + ContainersUnderTest []Container `yaml:"containersUnderTest" json:"containersUnderTest"` + // ExcludeContainersFromConnectivityTests excludes specific containers from network connectivity tests. This is particularly useful for containers that don't have ping available. + ExcludeContainersFromConnectivityTests []ContainerIdentifier `yaml:"excludeContainersFromConnectivityTests" json:"excludeContainersFromConnectivityTests"` + // Operator is the list of operator objects that needs to be tested. + Operators []Operator `yaml:"operators,omitempty" json:"operators,omitempty"` +} diff --git a/pkg/config/configsections/container_test.go b/pkg/config/configsections/config_section_test.go similarity index 82% rename from pkg/config/configsections/container_test.go rename to pkg/config/configsections/config_section_test.go index 29cd7c31f..9131e0670 100644 --- a/pkg/config/configsections/container_test.go +++ b/pkg/config/configsections/config_section_test.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package configsections_test +package configsections import ( "encoding/json" @@ -24,8 +24,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/test-network-function/test-network-function/pkg/config" - "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "gopkg.in/yaml.v2" ) @@ -34,7 +32,7 @@ var ( file *os.File jsonFile *os.File err error - test config.File + test TestConfiguration ) const ( @@ -71,7 +69,7 @@ const ( filePerm = 0644 ) -func saveConfig(c *config.File, configPath string) (err error) { +func saveConfig(c *TestConfiguration, configPath string) (err error) { bytes, _ := yaml.Marshal(c) if err != nil { return @@ -80,7 +78,7 @@ func saveConfig(c *config.File, configPath string) (err error) { return } -func saveConfigAsJSON(c *config.File, configPath string) (err error) { +func saveConfigAsJSON(c *TestConfiguration, configPath string) (err error) { bytes, err := json.Marshal(c) if err != nil { return @@ -90,9 +88,9 @@ func saveConfigAsJSON(c *config.File, configPath string) (err error) { } // newConfig returns a new decoded TnfContainerOperatorTestConfig struct -func newConfig(configPath string) (*config.File, error) { +func newConfig(configPath string) (*TestConfiguration, error) { // Create config structure - conf := &config.File{} + conf := &TestConfiguration{} // Open config file if file, err = os.Open(configPath); err != nil { return nil, err @@ -107,53 +105,47 @@ func newConfig(configPath string) (*config.File, error) { return conf, nil } -func loadCnfConfig() { - // CNF only - test.CNFs = []configsections.Cnf{ +func loadPodConfig() { + test.PodsUnderTest = []Pod{ { Name: cnfName, Namespace: testNameSpace, Tests: []string{testcases.PrivilegedPod}, }, } - test.CnfAvailableTestCases = nil - for key := range testcases.CnfTestTemplateFileMap { - test.CnfAvailableTestCases = append(test.CnfAvailableTestCases, key) - } } func loadOperatorConfig() { - operator := configsections.Operator{} + operator := Operator{} operator.Name = operatorName operator.Namespace = operatorNameSpace setCrdsAndInstances() - dep := configsections.Deployment{} + dep := Deployment{} dep.Name = deploymentName dep.Replicas = deploymentReplicas operator.Tests = []string{testcases.OperatorStatus} test.Operators = append(test.Operators, operator) - // CNF only - loadCnfConfig() + loadPodConfig() } func setCrdsAndInstances() { - crd := configsections.Crd{} + crd := Crd{} crd.Name = crdNameOne crd.Namespace = testNameSpace - instance := configsections.Instance{} + instance := Instance{} instance.Name = instanceNameOne crd.Instances = append(crd.Instances, instance) - crd2 := configsections.Crd{} + crd2 := Crd{} crd2.Name = crdNameTwo crd2.Namespace = testNameSpace - instance2 := configsections.Instance{} + instance2 := Instance{} instance2.Name = instanceNameTwo crd2.Instances = append(crd2.Instances, instance2) } func loadFullConfig() { loadOperatorConfig() - loadCnfConfig() + loadPodConfig() } func setup(configType string) { @@ -161,12 +153,12 @@ func setup(configType string) { if err != nil { log.Fatal(err) } - test = config.File{} + test = TestConfiguration{} switch configType { case fullConfig: loadFullConfig() case cnfConfig: - loadCnfConfig() + loadPodConfig() case operatorConfig: loadOperatorConfig() } @@ -181,12 +173,12 @@ func setupJSON(configType string) { if err != nil { log.Fatal(err) } - test = config.File{} + test = TestConfiguration{} switch configType { case fullConfig: loadFullConfig() case cnfConfig: - loadCnfConfig() + loadPodConfig() case operatorConfig: loadOperatorConfig() } @@ -211,16 +203,16 @@ func TestFullConfigLoad(t *testing.T) { cfg, err := newConfig(file.Name()) assert.NotNil(t, cfg) assert.Equal(t, len(cfg.Operators), 1) - assert.Equal(t, cfg.CNFs[0].Name, cnfName) + assert.Equal(t, cfg.PodsUnderTest[0].Name, cnfName) assert.Nil(t, err) } -func TestCnfConfigLoad(t *testing.T) { +func TestPodConfigLoad(t *testing.T) { setup(cnfConfig) defer (teardown)() cfg, err := newConfig(file.Name()) assert.NotNil(t, cfg) - assert.Equal(t, cfg.CNFs[0].Name, cnfName) + assert.Equal(t, cfg.PodsUnderTest[0].Name, cnfName) assert.Nil(t, err) } @@ -246,7 +238,7 @@ func TestFullJsonConfig(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, yamlCfg) assert.Equal(t, yamlCfg.Operators, jsonCfg.Operators) - assert.Equal(t, yamlCfg.CNFs, jsonCfg.CNFs) + assert.Equal(t, yamlCfg.PodsUnderTest, jsonCfg.PodsUnderTest) } func TestCnfJsonConfig(t *testing.T) { @@ -261,7 +253,7 @@ func TestCnfJsonConfig(t *testing.T) { yamlCfg, err := newConfig(file.Name()) assert.Nil(t, err) assert.NotNil(t, yamlCfg) - assert.Equal(t, yamlCfg.CNFs, jsonCfg.CNFs) + assert.Equal(t, yamlCfg.PodsUnderTest, jsonCfg.PodsUnderTest) } func TestOperatorJsonConfig(t *testing.T) { diff --git a/pkg/config/configsections/generic_config.go b/pkg/config/configsections/container.go similarity index 63% rename from pkg/config/configsections/generic_config.go rename to pkg/config/configsections/container.go index 151f1dba2..39e1356cb 100644 --- a/pkg/config/configsections/generic_config.go +++ b/pkg/config/configsections/container.go @@ -31,13 +31,3 @@ type Container struct { // MultusIPAddresses are the overlay IPs. MultusIPAddresses []string `yaml:"multusIpAddresses" json:"multusIpAddresses"` } - -// TestConfiguration provides generic test related configuration -type TestConfiguration struct { - ContainersUnderTest []Container `yaml:"containersUnderTest" json:"containersUnderTest"` - PartnerContainers []Container `yaml:"partnerContainers" json:"partnerContainers"` - TestOrchestrator ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` - FsDiffMasterContainer ContainerIdentifier `yaml:"fsDiffMasterContainer" json:"fsDiffMasterContainer"` - // ExcludeContainersFromConnectivityTests excludes specific containers from network connectivity tests. This is particularly useful for containers that don't have ping available. - ExcludeContainersFromConnectivityTests []ContainerIdentifier `yaml:"excludeContainersFromConnectivityTests" json:"excludeContainersFromConnectivityTests"` -} diff --git a/pkg/config/configsections/container_config.go b/pkg/config/configsections/misc.go similarity index 52% rename from pkg/config/configsections/container_config.go rename to pkg/config/configsections/misc.go index fda54b767..001f82e54 100644 --- a/pkg/config/configsections/container_config.go +++ b/pkg/config/configsections/misc.go @@ -16,46 +16,12 @@ package configsections +// Types defined in this file are not currently in use. Move them out when starting to use. +// May remove this altogether in the future + // CNFType defines a type to be either Operator or Container type CNFType string -// CertifiedContainerRequestInfo contains all certified images request info -type CertifiedContainerRequestInfo struct { - // Name is the name of the `operator bundle package name` or `image-version` that you want to check if exists in the RedHat catalog - Name string `yaml:"name" json:"name"` - - // Repository is the name of the repository `rhel8` of the container - // This is valid for container only and required field - Repository string `yaml:"repository" json:"repository"` -} - -// CertifiedOperatorRequestInfo contains all certified operator request info -type CertifiedOperatorRequestInfo struct { - - // Name is the name of the `operator bundle package name` that you want to check if exists in the RedHat catalog - Name string `yaml:"name" json:"name"` - - // Organization as understood by the operator publisher , e.g. `redhat-marketplace` - Organization string `yaml:"organization" json:"organization"` -} - -// Operator struct defines operator manifest for testing -type Operator struct { - - // Name is a required field, Name of the csv . - Name string `yaml:"name" json:"name"` - - // Namespace is a required field , namespace is where the csv is installed. - // If its all namespace then you can replace it with ALL_NAMESPACE TODO: add check for ALL_NAMESPACE - Namespace string `yaml:"namespace" json:"namespace"` - - // Tests this is list of test that need to run against the operator. - Tests []string `yaml:"tests" json:"tests"` - - // Subscription name is required field, Name of used subscription. - SubscriptionName string `yaml:"subscriptionName" json:"subscriptionName"` -} - // Crd struct defines Custom Resource Definition of the operator type Crd struct { // Name is the name of the CRD populated by the operator config generator @@ -86,18 +52,6 @@ type Permission struct { Role string `yaml:"role" json:"role"` } -// Cnf defines cloud network function in the cluster -type Cnf struct { - // Name is the name of a single Pod to test - Name string `yaml:"name" json:"name"` - - // Namespace where the Pod is deployed - Namespace string `yaml:"namespace" json:"namespace"` - - // Tests this is list of test that need to run against the Pod. - Tests []string `yaml:"tests" json:"tests"` -} - // Instance defines crd instances in the cluster type Instance struct { // Name is the name of the instance of custom resource (Auto populated) diff --git a/pkg/config/configsections/pod.go b/pkg/config/configsections/pod.go new file mode 100644 index 000000000..3b74956af --- /dev/null +++ b/pkg/config/configsections/pod.go @@ -0,0 +1,29 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +// Pod defines cloud network function in the cluster +type Pod struct { + // Name is the name of a single Pod to test + Name string `yaml:"name" json:"name"` + + // Namespace where the Pod is deployed + Namespace string `yaml:"namespace" json:"namespace"` + + // Tests this is list of test that need to run against the Pod. + Tests []string `yaml:"tests" json:"tests"` +} diff --git a/pkg/config/configsections/request.go b/pkg/config/configsections/request.go new file mode 100644 index 000000000..780b10abe --- /dev/null +++ b/pkg/config/configsections/request.go @@ -0,0 +1,37 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +// CertifiedContainerRequestInfo contains all certified images request info +type CertifiedContainerRequestInfo struct { + // Name is the name of the `operator bundle package name` or `image-version` that you want to check if exists in the RedHat catalog + Name string `yaml:"name" json:"name"` + + // Repository is the name of the repository `rhel8` of the container + // This is valid for container only and required field + Repository string `yaml:"repository" json:"repository"` +} + +// CertifiedOperatorRequestInfo contains all certified operator request info +type CertifiedOperatorRequestInfo struct { + + // Name is the name of the `operator bundle package name` that you want to check if exists in the RedHat catalog + Name string `yaml:"name" json:"name"` + + // Organization as understood by the operator publisher , e.g. `redhat-marketplace` + Organization string `yaml:"organization" json:"organization"` +} diff --git a/pkg/config/testdata/tnf_test_config.yml b/pkg/config/testdata/tnf_test_config.yml index 2833911b2..2867e0014 100644 --- a/pkg/config/testdata/tnf_test_config.yml +++ b/pkg/config/testdata/tnf_test_config.yml @@ -1,4 +1,4 @@ -generic: +testTarget: containersUnderTest: - namespace: default podName: test @@ -12,6 +12,19 @@ generic: defaultNetworkDevice: eth0 multusIpAddresses: - 10.217.0.29 + operators: + - name: etcdoperator.v0.9.4 + namespace: default + autogenerate: false + tests: + - OPERATOR_STATUS + podsUnderTest: # FKA cnfs + - name: ubuntu + namespace: default + tests: + - PRIVILEGED_POD + - PRIVILEGED_ROLE +testPartner: partnerContainers: - namespace: default podName: partner @@ -20,21 +33,9 @@ generic: multusIpAddresses: - 10.217.0.29 testOrchestrator: - namespace: default - podName: partner - containerName: partner -operators: - - name: etcdoperator.v0.9.4 - namespace: default - autogenerate: false - tests: - - OPERATOR_STATUS -cnfs: - - name: ubuntu - namespace: default - tests: - - PRIVILEGED_POD - - PRIVILEGED_ROLE + namespace: default + podName: partner + containerName: partner certifiedcontainerinfo: - name: nginx-116 # working example repository: rhel8 diff --git a/pkg/tnf/handlers/container/pod.go b/pkg/tnf/handlers/container/pod.go index 63ef5a8c1..81d85df13 100644 --- a/pkg/tnf/handlers/container/pod.go +++ b/pkg/tnf/handlers/container/pod.go @@ -138,12 +138,12 @@ func (p *Pod) ReelTimeout() *reel.Step { func (p *Pod) ReelEOF() { } -// Facts collects facts of the container +// Facts collects facts of the pod func (p *Pod) Facts() string { return p.facts } -// NewPod creates a `Container` test on the configured test cases. +// NewPod creates a `Pod` test on the configured test cases. func NewPod(args []string, name, namespace string, expectedStatus []string, resultType testcases.TestResultType, action testcases.TestAction, timeout time.Duration) *Pod { return &Pod{ Name: name, diff --git a/pkg/tnf/testcases/base.go b/pkg/tnf/testcases/base.go index 254c5725e..2b9d58c99 100644 --- a/pkg/tnf/testcases/base.go +++ b/pkg/tnf/testcases/base.go @@ -90,24 +90,24 @@ const ( OperatorStatus = "OPERATOR_STATUS" ) -// ContainerFactType type to hold container fact types -type ContainerFactType string +// PodFactType type to hold container fact types +type PodFactType string const ( // ServiceAccountName - for k8s service account name - ServiceAccountName ContainerFactType = "SERVICE_ACCOUNT_NAME" + ServiceAccountName PodFactType = "SERVICE_ACCOUNT_NAME" // Name for pod name - Name ContainerFactType = "NAME" + Name PodFactType = "NAME" // NameSpace for pod namespace - NameSpace ContainerFactType = "NAMESPACE" + NameSpace PodFactType = "NAMESPACE" // ClusterRole for cluster roles - ClusterRole ContainerFactType = "CLUSTER_ROLE" + ClusterRole PodFactType = "CLUSTER_ROLE" // ContainerCount for count of containers in the pod - ContainerCount ContainerFactType = "CONTAINER_COUNT" + ContainerCount PodFactType = "CONTAINER_COUNT" ) -// ContainerFact struct to store pod facts -type ContainerFact struct { +// PodFact struct to store pod facts +type PodFact struct { // Name of the pod under test Name string // Namespace of the pod under test @@ -122,8 +122,8 @@ type ContainerFact struct { Exists bool } -// CnfTestTemplateDataMap is map of available json data test case templates -var CnfTestTemplateDataMap = map[string]string{ +// PodTestTemplateDataMap is map of available json data test case templates +var PodTestTemplateDataMap = map[string]string{ GatherFacts: cnf.GatherPodFactsJSON, PrivilegedPod: cnf.PrivilegedPodJSON, PrivilegedRoles: cnf.RolesJSON, @@ -273,7 +273,7 @@ func ContainsConfiguredTest(a []ConfiguredTest, testType string) ConfiguredTest // LoadCnfTestCaseSpecs loads base test template data into a struct func LoadCnfTestCaseSpecs(name string) (*BaseTestCaseConfigSpec, error) { var testCaseConfigSpec BaseTestCaseConfigSpec - err := json.Unmarshal([]byte(CnfTestTemplateDataMap[name]), &testCaseConfigSpec) + err := json.Unmarshal([]byte(PodTestTemplateDataMap[name]), &testCaseConfigSpec) if err != nil { return nil, err } diff --git a/pkg/tnf/testcases/base_test.go b/pkg/tnf/testcases/base_test.go index c022dab4e..6cb8f904d 100644 --- a/pkg/tnf/testcases/base_test.go +++ b/pkg/tnf/testcases/base_test.go @@ -122,7 +122,7 @@ func TestLoadInvalidPathCNFTestCaseSpecsFromFile(t *testing.T) { } func TestBaseTestCase_CNFExpectedStatusFn(t *testing.T) { - var facts = testcases.ContainerFact{} + var facts = testcases.PodFact{} facts.Name = name facts.ServiceAccount = "TEST_SERVICE_ACCOUNT_NAME" testCase, err := testcases.LoadTestCaseSpecsFromFile(testcases.PrivilegedRoles, cnfFilePath, testcases.Cnf) diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index c11301d76..1b6d10e7b 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -58,47 +58,46 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { // Former "container" tests defer ginkgo.GinkgoRecover() - // Run the tests that interact with the containers + // Run the tests that interact with the pods ginkgo.When("under test", func() { conf := configpkg.GetConfigInstance() - cnfsInTest := conf.CNFs - gomega.Expect(cnfsInTest).ToNot(gomega.BeNil()) - for _, cnf := range cnfsInTest { - cnf := cnf - var containerFact = testcases.ContainerFact{Namespace: cnf.Namespace, Name: cnf.Name, ContainerCount: 0, HasClusterRole: false, Exists: true} + podsUnderTest := conf.PodsUnderTest + gomega.Expect(podsUnderTest).ToNot(gomega.BeNil()) + for _, pod := range podsUnderTest { + var podFact = testcases.PodFact{Namespace: pod.Namespace, Name: pod.Name, ContainerCount: 0, HasClusterRole: false, Exists: true} // Gather facts for containers podFacts, err := testcases.LoadCnfTestCaseSpecs(testcases.GatherFacts) gomega.Expect(err).To(gomega.BeNil()) context := common.GetContext() // Collect container facts for _, factsTest := range podFacts.TestCase { - args := strings.Split(fmt.Sprintf(factsTest.Command, cnf.Name, cnf.Namespace), " ") - cnfInTest := containerpkg.NewPod(args, cnf.Name, cnf.Namespace, factsTest.ExpectedStatus, factsTest.ResultType, factsTest.Action, common.DefaultTimeout) - test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) + args := strings.Split(fmt.Sprintf(factsTest.Command, pod.Name, pod.Namespace), " ") + podTest := containerpkg.NewPod(args, pod.Name, pod.Namespace, factsTest.ExpectedStatus, factsTest.ResultType, factsTest.Action, common.DefaultTimeout) + test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) _, err = test.Run() gomega.Expect(err).To(gomega.BeNil()) if factsTest.Name == string(testcases.ContainerCount) { - containerFact.ContainerCount, _ = strconv.Atoi(cnfInTest.Facts()) + podFact.ContainerCount, _ = strconv.Atoi(podTest.Facts()) } else if factsTest.Name == string(testcases.ServiceAccountName) { - containerFact.ServiceAccount = cnfInTest.Facts() + podFact.ServiceAccount = podTest.Facts() } else if factsTest.Name == string(testcases.Name) { - containerFact.Name = cnfInTest.Facts() - gomega.Expect(containerFact.Name).To(gomega.Equal(cnf.Name)) - if strings.Compare(containerFact.Name, cnf.Name) > 0 { - containerFact.Exists = true + podFact.Name = podTest.Facts() + gomega.Expect(podFact.Name).To(gomega.Equal(pod.Name)) + if strings.Compare(podFact.Name, pod.Name) > 0 { + podFact.Exists = true } } } // loop through various cnfs test - if !containerFact.Exists { - ginkgo.It(fmt.Sprintf("is running test pod exists : %s/%s for test command : %s", containerFact.Namespace, containerFact.Name, "POD EXISTS"), func() { - gomega.Expect(containerFact.Exists).To(gomega.BeTrue()) + if !podFact.Exists { + ginkgo.It(fmt.Sprintf("is running test pod exists : %s/%s for test command : %s", podFact.Namespace, podFact.Name, "POD EXISTS"), func() { + gomega.Expect(podFact.Exists).To(gomega.BeTrue()) }) continue } - for _, testType := range cnf.Tests { + for _, testType := range pod.Tests { testFile, err := testcases.LoadConfiguredTestFile(common.ConfiguredTestFile) gomega.Expect(testFile).ToNot(gomega.BeNil()) gomega.Expect(err).To(gomega.BeNil()) @@ -110,13 +109,13 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { if !testCase.SkipTest { if testCase.ExpectedType == testcases.Function { for _, val := range testCase.ExpectedStatus { - testCase.ExpectedStatusFn(cnf.Name, testcases.StatusFunctionType(val)) + testCase.ExpectedStatusFn(pod.Name, testcases.StatusFunctionType(val)) } } if testCase.Loop > 0 { - runTestsOnCNF(containerFact.ContainerCount, testCase, testType, containerFact, context) + runTestsOnPod(podFact.ContainerCount, testCase, testType, podFact, context) } else { - runTestsOnCNF(testCase.Loop, testCase, testType, containerFact, context) + runTestsOnPod(testCase.Loop, testCase, testType, podFact, context) } } } @@ -128,8 +127,8 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { }) //nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. -func runTestsOnCNF(containerCount int, testCmd testcases.BaseTestCase, - testType string, facts testcases.ContainerFact, context *interactive.Context) { +func runTestsOnPod(containerCount int, testCmd testcases.BaseTestCase, + testType string, facts testcases.PodFact, context *interactive.Context) { ginkgo.It(fmt.Sprintf("is running test for : %s/%s for test command : %s", facts.Namespace, facts.Name, testCmd.Name), func() { defer results.RecordResult(identifiers.TestHostResourceIdentifier) containerCount := containerCount @@ -159,9 +158,9 @@ func runTestsOnCNF(containerCount int, testCmd testcases.BaseTestCase, } } else { cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") - cnfInTest := containerpkg.NewPod(cmdArgs, facts.Name, facts.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) - gomega.Expect(cnfInTest).ToNot(gomega.BeNil()) - test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) + podTest := containerpkg.NewPod(cmdArgs, facts.Name, facts.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) + gomega.Expect(podTest).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) testResult, err := test.Run() diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index e43fbf50d..b3aa5167f 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -153,7 +153,7 @@ func RunAndValidateTest(test *tnf.Test) { // GetTestConfiguration returns the cnf-certification-generic-tests test configuration. func GetTestConfiguration() *configsections.TestConfiguration { conf := config.GetConfigInstance() - return &conf.Generic + return &conf } // IsMinikube returns true when the env var is set, OCP only test would be skipped based on this flag diff --git a/test-network-function/testconfigure.yml b/test-network-function/testconfigure.yml index c40f068b5..6cd2c290b 100644 --- a/test-network-function/testconfigure.yml +++ b/test-network-function/testconfigure.yml @@ -16,5 +16,4 @@ operatortest: - name: "OPERATOR_STATUS" tests: - "CSV_INSTALLED" - - "SUBSCRIPTION_INSTALLED" - "CSV_SCC" \ No newline at end of file diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 83bed489c..11c1c346a 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -1,40 +1,56 @@ targetPodLabels: - # populate this list with the spec.selector.matchLabels from all deployment/replica under test - #- namespace: - # name: - # value: -generic: - containersUnderTest: - - namespace: tnf - podName: test - containerName: test - defaultNetworkDevice: eth0 - multusIpAddresses: - - 10.217.0.8 - partnerContainers: - - namespace: tnf - podName: partner - containerName: partner - defaultNetworkDevice: eth0 - multusIpAddresses: - - 10.217.0.29 - testOrchestrator: - namespace: tnf - podName: partner - containerName: partner -operators: - - name: etcdoperator.v0.9.4 - namespace: default - subscriptionName: etcd - autogenerate: false - tests: - - OPERATOR_STATUS -cnfs: - - name: ubuntu - namespace: default - tests: - - PRIVILEGED_POD - - PRIVILEGED_ROLE + - namespace: test-network-function.com + name: generic + value: target +# The following section does not require manual configuration as autodiscovery is on by default +# Containers and pods will be found through matching targetPodLabels. Operators will be found if +# labelled with "test-network-function.com/operator". Their subscription name will be read from +# annotation named "subscription_name". +# +# To add additional test targets, uncomment the section and configuree the values +# +# testTarget: +# containersUnderTest: +# - namespace: tnf +# podName: test +# containerName: test +# defaultNetworkDevice: eth0 +# multusIpAddresses: +# - 10.217.0.8 +# podsUnderTest: +# - name: test +# namespace: tnf +# operators: +# - name: etcdoperator.v0.9.4 +# namespace: default +# subscriptionName: etcd +# autogenerate: false + + +# The following section does not require manual configuration as autodiscovery is on by default. +# Partner pods deployed automatically from the cnf-test-partner-repo should have all the labels +# used by autodiscovery +# +# testPartner: +# partnerContainers: +# - namespace: tnf +# podName: partner +# containerName: partner +# defaultNetworkDevice: eth0 +# multusIpAddresses: +# - 10.217.0.29 +# - namespace: tnf +# podName: node-master +# containerName: master +# defaultNetworkDevice: eth0 +# fsDiffMasterContainer: +# namespace: tnf +# podName: node-master +# containerName: master +# testOrchestrator: +# namespace: tnf +# podName: partner +# containerName: partner certifiedcontainerinfo: - name: nginx-116 # working example repository: rhel8 From 04eb720ac6f8644f78e1ccd8e5e0afe80117c47f Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:43:36 +0200 Subject: [PATCH 003/344] Added scale in/out testing to lifecycle TS (#292) Added new Scaling handler to execute the "oc scale" command. After the scaling command is issued, the test needs to wait until all deployments are reported as ready. --- CATALOG.md | 18 +++ pkg/tnf/handlers/scaling/doc.go | 18 +++ pkg/tnf/handlers/scaling/scaling.go | 95 +++++++++++++++ pkg/tnf/handlers/scaling/scaling_test.go | 82 +++++++++++++ pkg/tnf/identifier/identifiers.go | 21 ++++ .../identifiers/identifiers.go | 15 +++ test-network-function/lifecycle/suite.go | 108 ++++++++++++++++++ 7 files changed, 357 insertions(+) create mode 100644 pkg/tnf/handlers/scaling/doc.go create mode 100644 pkg/tnf/handlers/scaling/scaling.go create mode 100644 pkg/tnf/handlers/scaling/scaling_test.go diff --git a/CATALOG.md b/CATALOG.md index 0134a1d1f..50929204f 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -134,6 +134,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period tests whether the terminationGracePeriod is CNF-specific, or if the default (30s) is utilized. This test is informative, and will not affect CNF Certification. In many cases, the default terminationGracePeriod is perfectly acceptable for a CNF. Result Type|informative Suggested Remediation|Choose a terminationGracePeriod that is appropriate for your given CNF. If the default (30s) is appropriate, then feel free to ignore this informative message. This test is meant to raise awareness around how Pods are terminated, and to suggest that a CNF is configured based on its requirements. In addition to a terminationGracePeriod, consider utilizing a termination hook in the case that your application requires special shutdown instructions. +### http://test-network-function.com/testcases/lifecycle/scaling + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/lifecycle/scaling tests that CNF deployments support scale in/out operations. First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s. +Result Type|normative +Suggested Remediation|Make sure CNF deployments/replica sets can scale in/out successfully. ### http://test-network-function.com/testcases/networking/icmpv4-connectivity Property|Description @@ -523,6 +531,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`cat`, `oc` +### http://test-network-function.com/tests/scaling +Property|Description +---|--- +Version|v1.0.0 +Description|A test to check the deployments scale in/out. The tests issues the oc scale command on a deployment for a given number of replicas and checks whether the command output is valid. +Result Type|normative +Intrusive|true +Modifications Persist After Test|false +Runtime Binaries Required|`oc` + ### http://test-network-function.com/tests/serviceaccount Property|Description ---|--- diff --git a/pkg/tnf/handlers/scaling/doc.go b/pkg/tnf/handlers/scaling/doc.go new file mode 100644 index 000000000..546450436 --- /dev/null +++ b/pkg/tnf/handlers/scaling/doc.go @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package scaling provides a test for deployments scale in/out +package scaling diff --git a/pkg/tnf/handlers/scaling/scaling.go b/pkg/tnf/handlers/scaling/scaling.go new file mode 100644 index 000000000..d7d9efa34 --- /dev/null +++ b/pkg/tnf/handlers/scaling/scaling.go @@ -0,0 +1,95 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package scaling + +import ( + "fmt" + "strings" + "time" + + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +const ( + ocCommand = "oc scale --replicas=%d deployment %s -n %s" + regex = "^deployment.*/%s scaled" +) + +// Scaling holds the Scaling handler parameters. +type Scaling struct { + result int + timeout time.Duration + args []string + regex string +} + +// NewScaling creates a new Scaling handler. +func NewScaling(timeout time.Duration, namespace, deploymentName string, replicaCount int) *Scaling { + command := fmt.Sprintf(ocCommand, replicaCount, deploymentName, namespace) + return &Scaling{ + timeout: timeout, + result: tnf.ERROR, + args: strings.Fields(command), + regex: fmt.Sprintf(regex, deploymentName), + } +} + +// Args returns the command line args for the test. +func (scaling *Scaling) Args() []string { + return scaling.args +} + +// GetIdentifier returns the tnf.Test specific identifiesa. +func (scaling *Scaling) GetIdentifier() identifier.Identifier { + return identifier.ScalingIdentifier +} + +// Timeout returns the timeout in seconds for the test. +func (scaling *Scaling) Timeout() time.Duration { + return scaling.timeout +} + +// Result returns the test result. +func (scaling *Scaling) Result() int { + return scaling.result +} + +// ReelFirst returns a step which expects the scale command output within the test timeout. +func (scaling *Scaling) ReelFirst() *reel.Step { + return &reel.Step{ + Execute: "", + Expect: []string{scaling.regex}, + Timeout: scaling.timeout, + } +} + +// ReelMatch does nothing, just set the test result as success. +func (scaling *Scaling) ReelMatch(_, _, match string) *reel.Step { + scaling.result = tnf.SUCCESS + return nil +} + +// ReelTimeout does nothing; no action is necessary upon timeout. +func (scaling *Scaling) ReelTimeout() *reel.Step { + return nil +} + +// ReelEOF does nothing; no action is necessary upon EOF. +func (scaling *Scaling) ReelEOF() { +} diff --git a/pkg/tnf/handlers/scaling/scaling_test.go b/pkg/tnf/handlers/scaling/scaling_test.go new file mode 100644 index 000000000..305f3c6f2 --- /dev/null +++ b/pkg/tnf/handlers/scaling/scaling_test.go @@ -0,0 +1,82 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package scaling_test + +import ( + "fmt" + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" +) + +func Test_NewScaling(t *testing.T) { + handler := scaling.NewScaling(testTimeoutDuration, testPodNamespace, testDeploymentName, testReplicaCount) + assert.NotNil(t, handler) + assert.Equal(t, testTimeoutDuration, handler.Timeout()) + assert.Equal(t, handler.Result(), tnf.ERROR) +} + +func Test_ReelFirstPositive(t *testing.T) { + handler := scaling.NewScaling(testTimeoutDuration, testPodNamespace, testDeploymentName, testReplicaCount) + assert.NotNil(t, handler) + firstStep := handler.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + + matches := re.FindStringSubmatch(testInputSuccess) + assert.Len(t, matches, 1) +} + +func Test_ReelFirstNegative(t *testing.T) { + handler := scaling.NewScaling(testTimeoutDuration, testPodNamespace, testDeploymentName, testReplicaCount) + assert.NotNil(t, handler) + firstStep := handler.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputError) + assert.Len(t, matches, 0) +} + +func Test_ReelMatchSuccess(t *testing.T) { + handler := scaling.NewScaling(testTimeoutDuration, testPodNamespace, testDeploymentName, testReplicaCount) + assert.NotNil(t, handler) + + step := handler.ReelMatch("", "", testInputSuccess) + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, handler.Result()) +} + +// Just ensure there are no panics. +func Test_ReelEof(t *testing.T) { + handler := scaling.NewScaling(testTimeoutDuration, testPodNamespace, testDeploymentName, testReplicaCount) + assert.NotNil(t, handler) + handler.ReelEOF() +} + +const ( + testTimeoutDuration = time.Second * 1 + testReplicaCount = 2 + testInputError = "" + testPodNamespace = "testPodNamespace" + testDeploymentName = "testDeploymentName" +) + +var ( + testInputSuccess = fmt.Sprintf("deployment.apps/%s scaled\n", testDeploymentName) +) diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 1baf1821e..7f842c71d 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -55,6 +55,7 @@ const ( loggingIdentifierURL = "http://test-network-function.com/tests/logging" podantiaffinityIdentifierURL = "http://test-network-function.com/tests/testPodHighAvailability" shutdownIdentifierURL = "http://test-network-function.com/tests/shutdown" + scalingIdentifierURL = "http://test-network-function.com/tests/scaling" versionOne = "v1.0.0" ) @@ -556,6 +557,20 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.OcBinaryName, }, }, + scalingIdentifierURL: { + Identifier: ScalingIdentifier, + Description: "A test to check the deployments scale in/out. The tests issues the oc scale " + + "command on a deployment for a given number of replicas and checks whether the command output " + + "is valid.", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: true, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + }, + }, } // HostnameIdentifier is the Identifier used to represent the generic hostname test case. @@ -773,3 +788,9 @@ var ShutdownURLIdentifier = Identifier{ URL: shutdownIdentifierURL, SemanticVersion: versionOne, } + +// ScalingIdentifier is the Identifier used to represent a test that checks deployments scale in/out +var ScalingIdentifier = Identifier{ + URL: scalingIdentifierURL, + SemanticVersion: versionOne, +} diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index d37eb7e42..613254eb9 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -179,6 +179,12 @@ var ( Url: formTestURL(common.LifecycleTestKey, "container-shutdown"), Version: versionOne, } + + // TestScalingIdentifier ensures deployment scale in/out operations work correctly. + TestScalingIdentifier = claim.Identifier{ + Url: formTestURL(common.LifecycleTestKey, "scaling"), + Version: versionOne, + } ) func formDescription(identifier claim.Identifier, description string) string { @@ -461,4 +467,13 @@ the changes for you.`, Remediation: `Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails.`, }, + TestScalingIdentifier: { + Identifier: TestScalingIdentifier, + Type: normativeResult, + Description: formDescription(TestScalingIdentifier, + `tests that CNF deployments support scale in/out operations. + First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the + scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s.`), + Remediation: `Make sure CNF deployments/replica sets can scale in/out successfully.`, + }, } diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 3d8fe7a40..007a7b8ce 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -20,9 +20,11 @@ import ( "fmt" "path" "sort" + "strings" "time" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -47,6 +49,8 @@ const ( defaultTerminationGracePeriod = 30 drainTimeoutMinutes = 5 partnerPod = "partner" + scalingTimeout = 60 * time.Second + scalingPollingPeriod = 1 * time.Second ) var ( @@ -101,12 +105,116 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { if !common.NonIntrusive() { testPodsRecreation(&configData) + + testScaling(&configData) } testOwner(&configData) } }) +func waitForAllDeploymentsReady(namespace string, timeout, pollingPeriod time.Duration) { + gomega.Eventually(func() []string { + _, notReadyDeployments := getDeployments(namespace) + return notReadyDeployments + }, timeout, pollingPeriod).Should(gomega.HaveLen(0)) +} + +// restoreDeployments is the last attempt to restore the original test deployments' replicaCount +func restoreDeployments(configData *common.ConfigurationData, nsDeployments *map[string]dp.DeploymentMap) { + for namespace, originalDeployments := range *nsDeployments { + // For each deployment in the namespace, get the current replicas and compare. + deployments, notReadyDeployments := getDeployments(namespace) + + if len(notReadyDeployments) > 0 { + // Wait until the deployment is ready + waitForAllDeploymentsReady(namespace, scalingTimeout, scalingPollingPeriod) + } + + for originalDeploymentName, originalDeployment := range originalDeployments { + deployment := deployments[originalDeploymentName] + + if deployment.Replicas == originalDeployment.Replicas { + continue + } + + // Try to scale to the original deployment's replicaCount. + runScalingTest(namespace, originalDeploymentName, originalDeployment.Replicas) + + configData.SetNeedsRefresh() + } + } +} + +// saveDeployment Stores the dp.Deployment data into a map of namespace -> deployments +func saveDeployment(nsDeployments map[string]dp.DeploymentMap, namespace, deploymentName string, deployment *dp.Deployment) { + deployments, namespaceExists := nsDeployments[namespace] + + if !namespaceExists { + deployments = dp.DeploymentMap{} + deployments[deploymentName] = *deployment + nsDeployments[namespace] = deployments + } else { + // In case the deploymentName already exists, it will be overwritten. + deployments[deploymentName] = *deployment + } +} + +// runScalingTest Runs a Scaling handler TC with a given replicaCount and waits for all the deployments to be ready. +func runScalingTest(namespace, deploymentName string, replicaCount int) { + handler := scaling.NewScaling(common.DefaultTimeout, namespace, deploymentName, replicaCount) + test, err := tnf.NewTest(common.GetContext().GetExpecter(), handler, []reel.Handler{handler}, common.GetContext().GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + common.RunAndValidateTest(test) + + // Wait until the deployment is ready + waitForAllDeploymentsReady(namespace, scalingTimeout, scalingPollingPeriod) +} + +func testScaling(configData *common.ConfigurationData) { + ginkgo.It("Testing deployment scaling", func() { + defer results.RecordResult(identifiers.TestScalingIdentifier) + + namespaceDeploymentsBackup := make(map[string]dp.DeploymentMap) + defer restoreDeployments(configData, &namespaceDeploymentsBackup) + + // Map to register the deployments that have been already tested + deploymentNames := make(map[string]bool) + + for _, cut := range configData.ContainersUnderTest { + namespace := cut.Oc.GetPodNamespace() + + // Get deployment name and check whether it was already tested. + // ToDo: Proper way (helper/handler) to do this. + podNameParts := strings.Split(cut.Oc.GetPodName(), "-") + deploymentName := podNameParts[0] + + if _, alreadyTested := deploymentNames[deploymentName]; alreadyTested { + continue + } + + // Save deployment data for deferred restoring in case something's wrong during the TC. + deployments, _ := getDeployments(namespace) + deployment := deployments[deploymentName] + saveDeployment(namespaceDeploymentsBackup, namespace, deploymentName, &deployment) + + replicaCount := deployment.Replicas + + // ScaleIn, removing one pod from the replicaCount + runScalingTest(namespace, deploymentName, (replicaCount - 1)) + + // Scaleout, restoring the original replicaCount number + runScalingTest(namespace, deploymentName, replicaCount) + + // Ensure next tests/test suites receive a refreshed config. + configData.SetNeedsRefresh() + + // Set this deployment as tested + deploymentNames[deploymentName] = true + } + }) +} + func testNodeSelector(configData *common.ConfigurationData) { ginkgo.It("Testing pod nodeSelector", func() { for _, cut := range configData.ContainersUnderTest { From 71d9b5e525901ec393a978b8383c9aec2af1c817 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 01:48:06 -0400 Subject: [PATCH 004/344] Bump github.com/onsi/gomega from 1.14.0 to 1.15.0 (#295) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jun Chen --- go.mod | 4 +--- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index df5724bc0..5f4fb70a9 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,13 @@ require ( github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f github.com/kr/pretty v0.2.1 // indirect github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.14.0 + github.com/onsi/gomega v1.15.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.39.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 7b91fff22..1932cd032 100644 --- a/go.sum +++ b/go.sum @@ -85,7 +85,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -223,8 +222,8 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= -github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -320,7 +319,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -332,7 +330,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -443,7 +440,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -513,8 +509,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From fb9c119f07293b51a1038ce8737f44a1f0c8e968 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 01:57:16 -0400 Subject: [PATCH 005/344] Bump google.golang.org/grpc from 1.39.0 to 1.39.1 (#294) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.39.0 to 1.39.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.39.0...v1.39.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jun Chen --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5f4fb70a9..9c14b8c82 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,6 @@ require ( github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - google.golang.org/grpc v1.39.0 + google.golang.org/grpc v1.39.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 1932cd032..e1792c5d9 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1 h1:f37vZbBVTiJ6jKG5mWz8ySOBxNqy6ViPgyhSdVnxF3E= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From cff53a89e599ae995650f128320db55cfcfe7304 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:47:32 -0400 Subject: [PATCH 006/344] Bump google.golang.org/grpc from 1.39.1 to 1.40.0 (#303) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.39.1 to 1.40.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.39.1...v1.40.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9c14b8c82..2c09bb922 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,6 @@ require ( github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - google.golang.org/grpc v1.39.1 + google.golang.org/grpc v1.40.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index e1792c5d9..055a310f4 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/a-h/generate v0.0.0-20190312091541-e59c34d33fb3/go.mod h1:traiLYQ0YD7qUMCdjo6/jSaJRPHXniX4HVs+PhEhYpc= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -55,6 +56,7 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -242,6 +244,7 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= @@ -605,8 +608,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.1 h1:f37vZbBVTiJ6jKG5mWz8ySOBxNqy6ViPgyhSdVnxF3E= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From fb9b4c826d6145230f4215b82ca5ab05b7810a11 Mon Sep 17 00:00:00 2001 From: Isaak Dorfman Date: Thu, 12 Aug 2021 19:57:05 +0300 Subject: [PATCH 007/344] CNFCERT-73 (#297) Co-authored-by: Jun Chen --- CATALOG.md | 18 +++++++++++++++++ pkg/tnf/dependencies/binaries.go | 3 +++ pkg/tnf/identifier/identifiers.go | 20 +++++++++++++++++++ .../identifiers/identifiers.go | 15 +++++++++++++- 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/CATALOG.md b/CATALOG.md index 50929204f..3e33ec62b 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -198,6 +198,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. +### http://test-network-function.com/testcases/platform-alteration/sysctl-config + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that no one has changed the node's sysctl configs after the node was created, the tests works by checking if the sysctl configs are consistent with the MachineConfig CR which defines how the node should be configured +Result Type|normative +Suggested Remediation|You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes ### http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel Property|Description @@ -561,6 +569,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`oc` +### http://test-network-function.com/tests/sysctlAllConfigsArgs +Property|Description +---|--- +Version|v1.0.0 +Description|A test used to find all sysctl configuration args +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`sysctl` + ### http://test-network-function.com/tests/sysctlConfigFilesList Property|Description ---|--- diff --git a/pkg/tnf/dependencies/binaries.go b/pkg/tnf/dependencies/binaries.go index 4fdd5d044..1774ab484 100644 --- a/pkg/tnf/dependencies/binaries.go +++ b/pkg/tnf/dependencies/binaries.go @@ -50,6 +50,9 @@ const ( // OcBinaryName is the name of the OpenShift CLI client command. OcBinaryName = "oc" + // SysctlBinaryName is the name of the Sysctl command. + SysctlBinaryName = "sysctl" + // PodmanBinaryName is the name of the podman tool. PodmanBinaryName = "podman" diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 7f842c71d..3f74c0a8e 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -48,6 +48,7 @@ const ( currentKernelCmdlineArgsIdentifierURL = "http://test-network-function.com/tests/currentKernelCmdlineArgs" grubKernelCmdlineArgsIdentifierURL = "http://test-network-function.com/tests/grubKernelCmdlineArgs" sysctlConfigFilesListIdentifierURL = "http://test-network-function.com/tests/sysctlConfigFilesList" + sysctlAllConfigsArgsURL = "http://test-network-function.com/tests/sysctlAllConfigsArgs" readRemoteFileIdentifierURL = "http://test-network-function.com/tests/readRemoteFile" uncordonNodeIdentifierURL = "http://test-network-function.com/tests/node/uncordon" checkSubscriptionIdentifierURL = "http://test-network-function.com/tests/operator/check-subscription" @@ -557,6 +558,18 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.OcBinaryName, }, }, + sysctlAllConfigsArgsURL: { + Identifier: SysctlAllConfigsArgsIdentifier, + Description: "A test used to find all sysctl configuration args", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.SysctlBinaryName, + }, + }, scalingIdentifierURL: { Identifier: ScalingIdentifier, Description: "A test to check the deployments scale in/out. The tests issues the oc scale " + @@ -789,6 +802,13 @@ var ShutdownURLIdentifier = Identifier{ SemanticVersion: versionOne, } +// SysctlAllConfigsArgsIdentifier is the Identifier used to represent a test that checks all args in all sysctl conf files ordered +// in the same way as they are loaded by the os +var SysctlAllConfigsArgsIdentifier = Identifier{ + URL: sysctlAllConfigsArgsURL, + SemanticVersion: versionOne, +} + // ScalingIdentifier is the Identifier used to represent a test that checks deployments scale in/out var ScalingIdentifier = Identifier{ URL: scalingIdentifierURL, diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 613254eb9..b443e3398 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -179,7 +179,11 @@ var ( Url: formTestURL(common.LifecycleTestKey, "container-shutdown"), Version: versionOne, } - + // TestSysctlConfigsIdentifier ensures that the node's sysctl configs are consistent with the MachineConfig CR + TestSysctlConfigsIdentifier = claim.Identifier{ + Url: formTestURL(common.PlatformAlterationTestKey, "sysctl-config"), + Version: versionOne, + } // TestScalingIdentifier ensures deployment scale in/out operations work correctly. TestScalingIdentifier = claim.Identifier{ Url: formTestURL(common.LifecycleTestKey, "scaling"), @@ -467,6 +471,15 @@ the changes for you.`, Remediation: `Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails.`, }, + TestSysctlConfigsIdentifier: { + Identifier: TestSysctlConfigsIdentifier, + Type: normativeResult, + Description: formDescription(TestPodRecreationIdentifier, + `tests that no one has changed the node's sysctl configs after the node + was created, the tests works by checking if the sysctl configs are consistent with the + MachineConfig CR which defines how the node should be configured`), + Remediation: `You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes`, + }, TestScalingIdentifier: { Identifier: TestScalingIdentifier, Type: normativeResult, From c9d322a895972f7697fba9e8e79b0e66cfc017ab Mon Sep 17 00:00:00 2001 From: Isaak Dorfman Date: Thu, 12 Aug 2021 20:16:56 +0300 Subject: [PATCH 008/344] Test boot param fix (#279) The changes have been merged with the same test under platform Co-authored-by: Jun Chen Co-authored-by: Salaheddine Hamadi --- .github/workflows/pre-main.yaml | 1 + .../bootconfigentries/bootconfigentries.go | 2 +- .../handlers/readbootconfig/readbootconfig.go | 5 +- .../readbootconfig/readbootconfig_test.go | 8 +- test-network-function/generic/suite.go | 124 ------------------ test-network-function/platform/suite.go | 33 +---- 6 files changed, 11 insertions(+), 162 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 6b43c9348..76c61e7d2 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -19,6 +19,7 @@ env: TNF_OUTPUT_DIR: /tmp/tnf/output TNF_SRC_URL: 'https://github.com/${{ github.repository }}' TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${IMAGE_NAME}:${IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}' + NAMESPACE_TO_GENERATE: tnf jobs: lint: diff --git a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go index 683782f7b..04f18925c 100644 --- a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go +++ b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go @@ -43,7 +43,7 @@ func NewBootConfigEntries(timeout time.Duration, nodeName string) *BootConfigEnt timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"ls /host/boot/loader/entries/\"", "|", "oc", "debug", "-q", "node/" + nodeName, + "echo", "\"ls /host/boot/loader/entries/\"", "|", "oc", "debug", "--preserve-pod=true", "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/readbootconfig/readbootconfig.go b/pkg/tnf/handlers/readbootconfig/readbootconfig.go index 75cb709fb..e8136e5d8 100644 --- a/pkg/tnf/handlers/readbootconfig/readbootconfig.go +++ b/pkg/tnf/handlers/readbootconfig/readbootconfig.go @@ -37,12 +37,13 @@ type ReadBootConfig struct { } // NewReadBootConfig creates a ReadBootConfig tnf.Test. -func NewReadBootConfig(timeout time.Duration, nodeName, entryName string) *ReadBootConfig { +func NewReadBootConfig(timeout time.Duration, nodeName /*, entryName*/ string) *ReadBootConfig { return &ReadBootConfig{ timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"cat /host/boot/loader/entries/" + entryName + "\"", "|", "oc", "debug", "-q", "node/" + nodeName, + // "echo", "\"cat /host/boot/loader/entries/" + entryName + "\"", "|", "oc", "debug", "--preserve-pod=true", "node/" + nodeName, + "echo", "\"cat /host/boot/loader/entries/\\`ls /host/boot/loader/entries/ | sort | tail -n 1\\`\"", "|", "oc", "debug", "-q", "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go b/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go index c1f1cec9a..f15163363 100644 --- a/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go +++ b/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go @@ -28,13 +28,13 @@ import ( ) func TestReadBootConfig(t *testing.T) { - newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName, testBootEntryName) + newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName) assert.NotNil(t, newReadBootConfig) assert.Equal(t, tnf.ERROR, newReadBootConfig.Result()) } func Test_ReelFirst(t *testing.T) { - newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName, testBootEntryName) + newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName) assert.NotNil(t, newReadBootConfig) firstStep := newReadBootConfig.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -65,6 +65,6 @@ const ( options random.trust_cpu=on console=tty0 linux /ostree/rhcos-8db99645874 initrd /ostree/rhcos-8db99645874` - testNodeName = "crc-l6qvn-master-0" - testBootEntryName = "ostree-2-rhcos.conf" + testNodeName = "crc-l6qvn-master-0" + // testBootEntryName = "ostree-2-rhcos.conf" ) diff --git a/test-network-function/generic/suite.go b/test-network-function/generic/suite.go index e05eaa3b2..26b154a59 100644 --- a/test-network-function/generic/suite.go +++ b/test-network-function/generic/suite.go @@ -17,16 +17,11 @@ package generic import ( - "encoding/json" "fmt" - "strconv" - "strings" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" - "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" @@ -34,15 +29,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/bootconfigentries" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/currentkernelcmdlineargs" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/mckernelarguments" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodemcname" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/podnodename" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/readbootconfig" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" - utils "github.com/test-network-function/test-network-function/pkg/utils" ) const ( @@ -73,14 +61,6 @@ var _ = ginkgo.Describe(testsKey, func() { testIsRedHatRelease(containerUnderTest.Oc) } testIsRedHatRelease(testOrchestrator.Oc) - - if !common.IsMinikube() { - for _, containersUnderTest := range containersUnderTest { - // To be removed once Isaac's fix is merged - testBootParams(common.GetContext(), containersUnderTest.Oc.GetPodName(), containersUnderTest.Oc.GetPodNamespace(), containersUnderTest.Oc) - } - } - } }) @@ -99,107 +79,3 @@ func testIsRedHatRelease(oc *interactive.Oc) { }) }) } - -func getMcKernelArguments(context *interactive.Context, mcName string) map[string]string { - mcKernelArgumentsTester := mckernelarguments.NewMcKernelArguments(common.DefaultTimeout, mcName) - test, err := tnf.NewTest(context.GetExpecter(), mcKernelArgumentsTester, []reel.Handler{mcKernelArgumentsTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - mcKernelArguments := mcKernelArgumentsTester.GetKernelArguments() - var mcKernelArgumentsJSON []string - err = json.Unmarshal([]byte(mcKernelArguments), &mcKernelArgumentsJSON) - gomega.Expect(err).To(gomega.BeNil()) - mcKernelArgumentsMap := utils.ArgListToMap(mcKernelArgumentsJSON) - return mcKernelArgumentsMap -} - -func getMcName(context *interactive.Context, nodeName string) string { - mcNameTester := nodemcname.NewNodeMcName(common.DefaultTimeout, nodeName) - test, err := tnf.NewTest(context.GetExpecter(), mcNameTester, []reel.Handler{mcNameTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - return mcNameTester.GetMcName() -} - -func getPodNodeName(context *interactive.Context, podName, podNamespace string) string { - podNameTester := podnodename.NewPodNodeName(common.DefaultTimeout, podName, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), podNameTester, []reel.Handler{podNameTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - return podNameTester.GetNodeName() -} - -func getCurrentKernelCmdlineArgs(targetPodOc *interactive.Oc) map[string]string { - currentKernelCmdlineArgsTester := currentkernelcmdlineargs.NewCurrentKernelCmdlineArgs(common.DefaultTimeout) - test, err := tnf.NewTest(targetPodOc.GetExpecter(), currentKernelCmdlineArgsTester, []reel.Handler{currentKernelCmdlineArgsTester}, targetPodOc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - currnetKernelCmdlineArgs := currentKernelCmdlineArgsTester.GetKernelArguments() - currentSplitKernelCmdlineArgs := strings.Split(currnetKernelCmdlineArgs, " ") - return utils.ArgListToMap(currentSplitKernelCmdlineArgs) -} - -func getBootEntryIndex(bootEntry string) (int, error) { - return strconv.Atoi(strings.Split(bootEntry, "-")[1]) -} - -func getMaxIndexEntry(bootConfigEntries []string) string { - maxIndex, err := getBootEntryIndex(bootConfigEntries[0]) - gomega.Expect(err).To(gomega.BeNil()) - maxIndexEntryName := bootConfigEntries[0] - for _, bootEntry := range bootConfigEntries { - if entryIndex, err2 := getBootEntryIndex(bootEntry); entryIndex > maxIndex { - maxIndex = entryIndex - gomega.Expect(err2).To(gomega.BeNil()) - maxIndexEntryName = bootEntry - } - } - - return maxIndexEntryName -} - -func getGrubKernelArgs(context *interactive.Context, nodeName string) map[string]string { - bootConfigEntriesTester := bootconfigentries.NewBootConfigEntries(common.DefaultTimeout, nodeName) - test, err := tnf.NewTest(context.GetExpecter(), bootConfigEntriesTester, []reel.Handler{bootConfigEntriesTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - bootConfigEntries := bootConfigEntriesTester.GetBootConfigEntries() - - maxIndexEntryName := getMaxIndexEntry(bootConfigEntries) - - readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout, nodeName, maxIndexEntryName) - test, err = tnf.NewTest(context.GetExpecter(), readBootConfigTester, []reel.Handler{readBootConfigTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - bootConfig := readBootConfigTester.GetBootConfig() - - splitBootConfig := strings.Split(bootConfig, "\n") - filteredBootConfig := utils.FilterArray(splitBootConfig, func(line string) bool { - return strings.HasPrefix(line, "options") - }) - gomega.Expect(len(filteredBootConfig)).To(gomega.Equal(1)) - grubKernelConfig := filteredBootConfig[0] - grubSplitKernelConfig := strings.Split(grubKernelConfig, " ") - grubSplitKernelConfig = grubSplitKernelConfig[1:] - return utils.ArgListToMap(grubSplitKernelConfig) -} - -func testBootParams(context *interactive.Context, podName, podNamespace string, targetPodOc *interactive.Oc) { - ginkgo.It(fmt.Sprintf("Testing boot params for the pod's node %s/%s", podNamespace, podName), func() { - defer results.RecordResult(identifiers.TestUnalteredStartupBootParamsIdentifier) - nodeName := getPodNodeName(context, podName, podNamespace) - mcName := getMcName(context, nodeName) - mcKernelArgumentsMap := getMcKernelArguments(context, mcName) - currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetPodOc) - grubKernelConfigMap := getGrubKernelArgs(context, nodeName) - - for key, mcVal := range mcKernelArgumentsMap { - if currentVal, ok := currentKernelArgsMap[key]; ok { - gomega.Expect(currentVal).To(gomega.Equal(mcVal)) - } - if grubVal, ok := grubKernelConfigMap[key]; ok { - gomega.Expect(grubVal).To(gomega.Equal(mcVal)) - } - } - }) -} diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index b1f9e65e5..dde598f4d 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "regexp" - "strconv" "strings" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -34,7 +33,6 @@ import ( "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/bootconfigentries" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/cnffsdiff" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/containerid" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/currentkernelcmdlineargs" @@ -158,36 +156,9 @@ func getCurrentKernelCmdlineArgs(targetPodOc *interactive.Oc) map[string]string return utils.ArgListToMap(currentSplitKernelCmdlineArgs) } -func getBootEntryIndex(bootEntry string) (int, error) { - return strconv.Atoi(strings.Split(bootEntry, "-")[1]) -} - -func getMaxIndexEntry(bootConfigEntries []string) string { - maxIndex, err := getBootEntryIndex(bootConfigEntries[0]) - gomega.Expect(err).To(gomega.BeNil()) - maxIndexEntryName := bootConfigEntries[0] - for _, bootEntry := range bootConfigEntries { - if entryIndex, err2 := getBootEntryIndex(bootEntry); entryIndex > maxIndex { - maxIndex = entryIndex - gomega.Expect(err2).To(gomega.BeNil()) - maxIndexEntryName = bootEntry - } - } - - return maxIndexEntryName -} - func getGrubKernelArgs(context *interactive.Context, nodeName string) map[string]string { - bootConfigEntriesTester := bootconfigentries.NewBootConfigEntries(common.DefaultTimeout, nodeName) - test, err := tnf.NewTest(context.GetExpecter(), bootConfigEntriesTester, []reel.Handler{bootConfigEntriesTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - bootConfigEntries := bootConfigEntriesTester.GetBootConfigEntries() - - maxIndexEntryName := getMaxIndexEntry(bootConfigEntries) - - readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout, nodeName, maxIndexEntryName) - test, err = tnf.NewTest(context.GetExpecter(), readBootConfigTester, []reel.Handler{readBootConfigTester}, context.GetErrorChannel()) + readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout, nodeName) + test, err := tnf.NewTest(context.GetExpecter(), readBootConfigTester, []reel.Handler{readBootConfigTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) common.RunAndValidateTest(test) bootConfig := readBootConfigTester.GetBootConfig() From 25d8720426e11f29856e80b888bae9244ff30ca8 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 12 Aug 2021 17:19:14 -0400 Subject: [PATCH 009/344] suites refactoring (#296) networking suite refactor observability suite refactor certification suite refactor generic suite refactor diagnostic suite refactor platform suite refactor --- .github/workflows/pre-main.yaml | 2 +- pkg/config/config.go | 8 +- pkg/tnf/interactive/oc.go | 12 ++ test-network-function/accesscontrol/suite.go | 143 +++++++++-------- test-network-function/certification/suite.go | 53 ++++--- test-network-function/common/env.go | 3 +- test-network-function/generic/suite.go | 64 ++++---- test-network-function/lifecycle/suite.go | 5 +- test-network-function/networking/suite.go | 92 ++++++----- test-network-function/observability/suite.go | 34 ++-- test-network-function/operator/suite.go | 80 +++++----- test-network-function/platform/suite.go | 155 +++++++++++-------- test-network-function/tnf_config.yml | 4 +- 13 files changed, 357 insertions(+), 298 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 76c61e7d2..831535a75 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -121,7 +121,7 @@ jobs: run: ./run-cnf-suites.sh diagnostic - name: 'Test: Run test suites' - run: ./run-cnf-suites.sh access-control lifecycle platform observablility networking + run: ./run-cnf-suites.sh access-control lifecycle platform observablility networking affiliated-certification # Perform smoke tests using a TNF container. diff --git a/pkg/config/config.go b/pkg/config/config.go index fdf81e810..32faedb7e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -80,16 +80,18 @@ func GetConfigInstance() configsections.TestConfiguration { log.Fatalf("unable to load configuration file: %s", err) } autodiscover.FillTestPartner(&configInstance.TestPartner) - discoverTestTargets() + doAutodiscover() } else if needsRefresh { - discoverTestTargets() + configInstance.TestPartner = configsections.TestPartner{} + doAutodiscover() } return configInstance } -func discoverTestTargets() { +func doAutodiscover() { if autodiscover.PerformAutoDiscovery() { configInstance.TestTarget = autodiscover.FindTestTarget(configInstance.TargetPodLabels) + autodiscover.FillTestPartner(&configInstance.TestPartner) } needsRefresh = false } diff --git a/pkg/tnf/interactive/oc.go b/pkg/tnf/interactive/oc.go index ad2416aee..d0e95fff4 100644 --- a/pkg/tnf/interactive/oc.go +++ b/pkg/tnf/interactive/oc.go @@ -40,6 +40,8 @@ type Oc struct { container string // namespace of the pod namespace string + // serviceAccountName of the pod + serviceAccountName string // timeout for commands run in expecter timeout time.Duration // options for experter, such as expect.Verbose(true) @@ -83,6 +85,16 @@ func (o *Oc) GetPodNamespace() string { return o.namespace } +// GetServiceAccountName extracts the serviceAccountName of the pod +func (o *Oc) GetServiceAccountName() string { + return o.serviceAccountName +} + +// SetServiceAccountName sets the serviceAccountName of the pod +func (o *Oc) SetServiceAccountName(serviceAccountName string) { + o.serviceAccountName = serviceAccountName +} + // GetTimeout returns the timeout for the expect.Expecter. func (o *Oc) GetTimeout() time.Duration { return o.timeout diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 1b6d10e7b..d050f95e4 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -41,19 +41,15 @@ import ( var _ = ginkgo.Describe(common.AccessControlTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AccessControlTestKey) { - config := common.GetTestConfiguration() - log.Infof("Test Configuration: %s", config) - - containersUnderTest := common.CreateContainersUnderTest(config) - log.Info(containersUnderTest) + configData := common.ConfigurationData{} + configData.SetNeedsRefresh() + ginkgo.BeforeEach(func() { + common.ReloadConfiguration(&configData) + }) - for _, containerUnderTest := range containersUnderTest { - testNamespace(containerUnderTest.Oc) - } + testNamespace(&configData) - for _, containerUnderTest := range containersUnderTest { - testRoles(containerUnderTest.Oc.GetPodName(), containerUnderTest.Oc.GetPodNamespace()) - } + testRoles(&configData) // Former "container" tests defer ginkgo.GinkgoRecover() @@ -122,7 +118,6 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { } } }) - } }) @@ -170,76 +165,94 @@ func runTestsOnPod(containerCount int, testCmd testcases.BaseTestCase, }) } -func testNamespace(oc *interactive.Oc) { - pod := oc.GetPodName() - container := oc.GetPodContainerName() - ginkgo.When(fmt.Sprintf("Reading namespace of %s/%s", pod, container), func() { +func testNamespace(configData *common.ConfigurationData) { + ginkgo.When("test deployment namespace", func() { ginkgo.It("Should not be 'default' and should not begin with 'openshift-'", func() { - defer results.RecordResult(identifiers.TestNamespaceBestPracticesIdentifier) - gomega.Expect(oc.GetPodNamespace()).To(gomega.Not(gomega.Equal("default"))) - gomega.Expect(oc.GetPodNamespace()).To(gomega.Not(gomega.HavePrefix("openshift-"))) + for _, cut := range configData.ContainersUnderTest { + podName := cut.Oc.GetPodName() + podNamespace := cut.Oc.GetPodNamespace() + ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestNamespaceBestPracticesIdentifier) + gomega.Expect(podNamespace).To(gomega.Not(gomega.Equal("default"))) + gomega.Expect(podNamespace).To(gomega.Not(gomega.HavePrefix("openshift-"))) + } }) }) } -func testRoles(podName, podNamespace string) { - var serviceAccountName string - ginkgo.When(fmt.Sprintf("Testing roles and privileges of %s/%s", podNamespace, podName), func() { - testServiceAccount(podName, podNamespace, &serviceAccountName) - testRoleBindings(podNamespace, &serviceAccountName) - testClusterRoleBindings(podNamespace, &serviceAccountName) - }) +func testRoles(configData *common.ConfigurationData) { + testServiceAccount(configData) + testRoleBindings(configData) + testClusterRoleBindings(configData) } -func testServiceAccount(podName, podNamespace string, serviceAccountName *string) { +func testServiceAccount(configData *common.ConfigurationData) { ginkgo.It("Should have a valid ServiceAccount name", func() { - defer results.RecordResult(identifiers.TestPodServiceAccountBestPracticesIdentifier) - context := common.GetContext() - tester := serviceaccount.NewServiceAccount(common.DefaultTimeout, podName, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - *serviceAccountName = tester.GetServiceAccountName() - gomega.Expect(*serviceAccountName).ToNot(gomega.BeEmpty()) + for _, cut := range configData.ContainersUnderTest { + context := common.GetContext() + podName := cut.Oc.GetPodName() + podNamespace := cut.Oc.GetPodNamespace() + ginkgo.By(fmt.Sprintf("Testing pod service account %s %s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestPodServiceAccountBestPracticesIdentifier) + tester := serviceaccount.NewServiceAccount(common.DefaultTimeout, podName, podNamespace) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + serviceAccountName := tester.GetServiceAccountName() + cut.Oc.SetServiceAccountName(serviceAccountName) + gomega.Expect(serviceAccountName).ToNot(gomega.BeEmpty()) + } }) } -func testRoleBindings(podNamespace string, serviceAccountName *string) { +func testRoleBindings(configData *common.ConfigurationData) { ginkgo.It("Should not have RoleBinding in other namespaces", func() { - defer results.RecordResult(identifiers.TestPodRoleBindingsBestPracticesIdentifier) - if *serviceAccountName == "" { - ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") - } - context := common.GetContext() - rbTester := rolebinding.NewRoleBinding(common.DefaultTimeout, *serviceAccountName, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), rbTester, []reel.Handler{rbTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - if rbTester.Result() == tnf.FAILURE { - log.Info("RoleBindings: ", rbTester.GetRoleBindings()) + for _, cut := range configData.ContainersUnderTest { + context := common.GetContext() + podName := cut.Oc.GetPodName() + podNamespace := cut.Oc.GetPodNamespace() + serviceAccountName := cut.Oc.GetServiceAccountName() + defer results.RecordResult(identifiers.TestPodRoleBindingsBestPracticesIdentifier) + ginkgo.By(fmt.Sprintf("Testing role bidning %s %s", podNamespace, podName)) + if serviceAccountName == "" { + ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") + } + rbTester := rolebinding.NewRoleBinding(common.DefaultTimeout, serviceAccountName, podNamespace) + test, err := tnf.NewTest(context.GetExpecter(), rbTester, []reel.Handler{rbTester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + if rbTester.Result() == tnf.FAILURE { + log.Info("RoleBindings: ", rbTester.GetRoleBindings()) + } + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) } - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) }) } -func testClusterRoleBindings(podNamespace string, serviceAccountName *string) { +func testClusterRoleBindings(configData *common.ConfigurationData) { ginkgo.It("Should not have ClusterRoleBindings", func() { - defer results.RecordResult(identifiers.TestPodClusterRoleBindingsBestPracticesIdentifier) - if *serviceAccountName == "" { - ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") - } - context := common.GetContext() - crbTester := clusterrolebinding.NewClusterRoleBinding(common.DefaultTimeout, *serviceAccountName, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), crbTester, []reel.Handler{crbTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - if crbTester.Result() == tnf.FAILURE { - log.Info("ClusterRoleBindings: ", crbTester.GetClusterRoleBindings()) + for _, cut := range configData.ContainersUnderTest { + context := common.GetContext() + podName := cut.Oc.GetPodName() + podNamespace := cut.Oc.GetPodNamespace() + serviceAccountName := cut.Oc.GetServiceAccountName() + defer results.RecordResult(identifiers.TestPodClusterRoleBindingsBestPracticesIdentifier) + ginkgo.By(fmt.Sprintf("Testing cluster role bidning %s %s", podNamespace, podName)) + if serviceAccountName == "" { + ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") + } + crbTester := clusterrolebinding.NewClusterRoleBinding(common.DefaultTimeout, serviceAccountName, podNamespace) + test, err := tnf.NewTest(context.GetExpecter(), crbTester, []reel.Handler{crbTester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + if crbTester.Result() == tnf.FAILURE { + log.Info("ClusterRoleBindings: ", crbTester.GetClusterRoleBindings()) + } + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) } - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) }) } diff --git a/test-network-function/certification/suite.go b/test-network-function/certification/suite.go index 2e23a9b39..8e7385daf 100644 --- a/test-network-function/certification/suite.go +++ b/test-network-function/certification/suite.go @@ -41,43 +41,50 @@ var certAPIClient api.CertAPIClient var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AffiliatedCertTestKey) { + return + } + testContainerCertificationStatus() + testOperatorCertificationStatus() +}) - // Query API for certification status of listed containers - ginkgo.When("getting certification status", func() { +func testContainerCertificationStatus() { + // Query API for certification status of listed containers + ginkgo.When("getting certification status", func() { + ginkgo.It("get certification status", func() { + defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) conf := configpkg.GetConfigInstance() cnfsToQuery := conf.CertifiedContainerInfo if len(cnfsToQuery) > 0 { certAPIClient = api.NewHTTPClient() - for _, cnfRequestInfo := range cnfsToQuery { - cnf := cnfRequestInfo + for _, cnf := range cnfsToQuery { + cnf := cnf // pin // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. - ginkgo.It(fmt.Sprintf("container %s/%s should eventually be verified as certified", cnf.Repository, cnf.Name), func() { - defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) - cnf := cnf // pin - gomega.Eventually(func() bool { - isCertified := certAPIClient.IsContainerCertified(cnf.Repository, cnf.Name) - return isCertified - }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) - }) + ginkgo.By(fmt.Sprintf("container %s/%s should eventually be verified as certified", cnf.Repository, cnf.Name)) + gomega.Eventually(func() bool { + isCertified := certAPIClient.IsContainerCertified(cnf.Repository, cnf.Name) + return isCertified + }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) } } }) + }) +} +func testOperatorCertificationStatus() { + ginkgo.It("Verify operator as certified", func() { + defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) operatorsToQuery := configpkg.GetConfigInstance().CertifiedOperatorInfo if len(operatorsToQuery) > 0 { certAPIClient := api.NewHTTPClient() for _, certified := range operatorsToQuery { + ginkgo.By(fmt.Sprintf("should eventually be verified as certified (operator %s/%s)", certified.Organization, certified.Name)) // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. - ginkgo.It(fmt.Sprintf("should eventually be verified as certified (operator %s/%s)", certified.Organization, certified.Name), func() { - defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) - certified := certified // pin - gomega.Eventually(func() bool { - isCertified := certAPIClient.IsOperatorCertified(certified.Organization, certified.Name) - return isCertified - }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) - }) + certified := certified // pin + gomega.Eventually(func() bool { + isCertified := certAPIClient.IsOperatorCertified(certified.Organization, certified.Name) + return isCertified + }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) } } - - } -}) + }) +} diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index b3aa5167f..bb62ea9c9 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -174,7 +174,8 @@ type ConfigurationData struct { PartnerContainers map[configsections.ContainerIdentifier]*Container TestOrchestrator *Container FsDiffContainer *Container - needsRefresh bool + + needsRefresh bool } // createContainersUnderTest sets up the test containers. diff --git a/test-network-function/generic/suite.go b/test-network-function/generic/suite.go index 26b154a59..b117e9dae 100644 --- a/test-network-function/generic/suite.go +++ b/test-network-function/generic/suite.go @@ -19,6 +19,9 @@ package generic import ( "fmt" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -26,11 +29,6 @@ import ( "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" - "github.com/test-network-function/test-network-function/pkg/tnf/interactive" - "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) const ( @@ -44,38 +42,36 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(testsKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testsKey) { + configData := common.ConfigurationData{} + configData.SetNeedsRefresh() + ginkgo.BeforeEach(func() { + common.ReloadConfiguration(&configData) + }) - config := common.GetTestConfiguration() - log.Infof("Test Configuration: %s", config) - - for _, cid := range config.ExcludeContainersFromConnectivityTests { - common.ContainersToExcludeFromConnectivityTests[cid] = "" - } - containersUnderTest := common.CreateContainersUnderTest(config) - partnerContainers := common.CreatePartnerContainers(config) - testOrchestrator := partnerContainers[config.TestOrchestrator] - log.Info(testOrchestrator) - log.Info(containersUnderTest) - - for _, containerUnderTest := range containersUnderTest { - testIsRedHatRelease(containerUnderTest.Oc) - } - testIsRedHatRelease(testOrchestrator.Oc) + testIsRedHatRelease(&configData) } }) -// testIsRedHatRelease tests whether the container attached to oc is Red Hat based. -func testIsRedHatRelease(oc *interactive.Oc) { - pod := oc.GetPodName() - container := oc.GetPodContainerName() - ginkgo.When(fmt.Sprintf("%s(%s) is checked for Red Hat version", pod, container), func() { - ginkgo.It("Should report a proper Red Hat version", func() { - versionTester := redhat.NewRelease(common.DefaultTimeout) - test, err := tnf.NewTest(oc.GetExpecter(), versionTester, []reel.Handler{versionTester}, oc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - }) +// testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. +func testIsRedHatRelease(configData *common.ConfigurationData) { + ginkgo.It("Should report a proper Red Hat version", func() { + for _, cut := range configData.ContainersUnderTest { + testContainerIsRedHatRelease(cut) + } + testContainerIsRedHatRelease(configData.TestOrchestrator) }) } + +// testContainerIsRedHatRelease tests whether the container attached to oc is Red Hat based. +func testContainerIsRedHatRelease(cut *common.Container) { + podName := cut.Oc.GetPodName() + containerName := cut.Oc.GetPodContainerName() + context := cut.Oc + ginkgo.By(fmt.Sprintf("%s(%s) is checked for Red Hat version", podName, containerName)) + versionTester := redhat.NewRelease(common.DefaultTimeout) + test, err := tnf.NewTest(context.GetExpecter(), versionTester, []reel.Handler{versionTester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) +} diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 007a7b8ce..67a1c2fc0 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -86,10 +86,7 @@ var drainTimeout = time.Duration(drainTimeoutMinutes) * time.Minute // var _ = ginkgo.Describe(common.LifecycleTestKey, func() { configData := common.ConfigurationData{} - ginkgo.BeforeSuite(func() { - common.Loadconfiguration(&configData) - log.Info(configData.ContainersUnderTest) - }) + configData.SetNeedsRefresh() ginkgo.BeforeEach(func() { common.ReloadConfiguration(&configData) }) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 4f81e04b3..74f6c3100 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -46,53 +46,63 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(common.NetworkingTestKey, func() { + configData := common.ConfigurationData{} + configData.SetNeedsRefresh() + ginkgo.BeforeEach(func() { + common.ReloadConfiguration(&configData) + }) if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.NetworkingTestKey) { - config := common.GetTestConfiguration() - log.Infof("Test Configuration: %s", config) - - for _, cid := range config.ExcludeContainersFromConnectivityTests { - common.ContainersToExcludeFromConnectivityTests[cid] = "" - } - containersUnderTest := common.CreateContainersUnderTest(config) - partnerContainers := common.CreatePartnerContainers(config) - testOrchestrator := partnerContainers[config.TestOrchestrator] - log.Info(testOrchestrator) - log.Info(containersUnderTest) - ginkgo.Context("Both Pods are on the Default network", func() { // for each container under test, ensure bidirectional ICMP traffic between the container and the orchestrator. - for _, containerUnderTest := range containersUnderTest { - if _, ok := common.ContainersToExcludeFromConnectivityTests[containerUnderTest.ContainerIdentifier]; !ok { - testNetworkConnectivity(containerUnderTest.Oc, testOrchestrator.Oc, testOrchestrator.DefaultNetworkIPAddress, defaultNumPings) - testNetworkConnectivity(testOrchestrator.Oc, containerUnderTest.Oc, containerUnderTest.DefaultNetworkIPAddress, defaultNumPings) - } - } + testDefaultNetworkConnectivity(&configData, defaultNumPings) }) ginkgo.Context("Both Pods are connected via a Multus Overlay Network", func() { // Unidirectional test; for each container under test, attempt to ping the target Multus IP addresses. - for _, containerUnderTest := range containersUnderTest { - for _, multusIPAddress := range containerUnderTest.ContainerConfiguration.MultusIPAddresses { - testNetworkConnectivity(testOrchestrator.Oc, containerUnderTest.Oc, multusIPAddress, defaultNumPings) - } - } + testMultusNetworkConnectivity(&configData, defaultNumPings) + }) + ginkgo.Context("Should not have type of nodePort", func() { + testNodePort(&configData) }) - - for _, containerUnderTest := range containersUnderTest { - testNodePort(containerUnderTest.Oc.GetPodNamespace()) - } - } }) -// Helper to test that a container can ping a target IP address, and report through Ginkgo. -func testNetworkConnectivity(initiatingPodOc, targetPodOc *interactive.Oc, targetPodIPAddress string, count int) { - ginkgo.When(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", initiatingPodOc.GetPodName(), - initiatingPodOc.GetPodContainerName(), targetPodOc.GetPodName(), targetPodOc.GetPodContainerName(), - targetPodIPAddress), func() { - ginkgo.It(fmt.Sprintf("%s(%s) should reply", targetPodOc.GetPodName(), targetPodOc.GetPodContainerName()), func() { - defer results.RecordResult(identifiers.TestICMPv4ConnectivityIdentifier) - testPing(initiatingPodOc, targetPodIPAddress, count) +func testDefaultNetworkConnectivity(configData *common.ConfigurationData, count int) { + ginkgo.When("Testing network connectivity", func() { + ginkgo.It("should reply to ping", func() { + for _, cut := range configData.ContainersUnderTest { + if _, ok := common.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { + continue + } + context := cut.Oc + testOrchestrator := configData.TestOrchestrator + ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), + testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), + cut.DefaultNetworkIPAddress)) + defer results.RecordResult(identifiers.TestICMPv4ConnectivityIdentifier) + testPing(testOrchestrator.Oc, cut.DefaultNetworkIPAddress, count) + ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", cut.Oc.GetPodName(), + cut.Oc.GetPodContainerName(), testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), + testOrchestrator.DefaultNetworkIPAddress)) + testPing(context, testOrchestrator.DefaultNetworkIPAddress, count) + } + }) + }) +} + +func testMultusNetworkConnectivity(configData *common.ConfigurationData, count int) { + ginkgo.When("Testing network connectivity", func() { + ginkgo.It("should reply to ping", func() { + for _, cut := range configData.ContainersUnderTest { + for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { + testOrchestrator := configData.TestOrchestrator + ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), + testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), + cut.DefaultNetworkIPAddress)) + defer results.RecordResult(identifiers.TestICMPv4ConnectivityIdentifier) + testPing(testOrchestrator.Oc, multusIPAddress, count) + } + } }) }) } @@ -109,17 +119,19 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count gomega.Expect(errors).To(gomega.BeZero()) } -func testNodePort(podNamespace string) { - ginkgo.When(fmt.Sprintf("Testing services in namespace %s", podNamespace), func() { - ginkgo.It("Should not have services of type NodePort", func() { +func testNodePort(configData *common.ConfigurationData) { + ginkgo.It("Should not have services of type NodePort", func() { + for _, cut := range configData.ContainersUnderTest { defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) context := common.GetContext() + podNamespace := cut.Oc.GetPodNamespace() + ginkgo.By(fmt.Sprintf("Testing services in namespace %s", podNamespace)) tester := nodeport.NewNodePort(common.DefaultTimeout, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) - }) + } }) } diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index b31378c43..362f5d347 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -17,12 +17,12 @@ package observability import ( + "fmt" "path" "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -44,27 +44,27 @@ var ( var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.ObservabilityTestKey) { - config := common.GetTestConfiguration() - log.Infof("Test Configuration: %s", config) - - for _, cid := range config.ExcludeContainersFromConnectivityTests { - common.ContainersToExcludeFromConnectivityTests[cid] = "" - } - containersUnderTest := common.CreateContainersUnderTest(config) - log.Info(containersUnderTest) - - for _, containerUnderTest := range containersUnderTest { - testLogging(containerUnderTest.Oc.GetPodNamespace(), containerUnderTest.Oc.GetPodName(), containerUnderTest.Oc.GetPodContainerName()) - } - + configData := common.ConfigurationData{} + configData.SetNeedsRefresh() + ginkgo.BeforeEach(func() { + common.ReloadConfiguration(&configData) + }) + testLogging(&configData) } }) -func testLogging(podNameSpace, podName, containerName string) { +func testLogging(configData *common.ConfigurationData) { ginkgo.When("Testing PUT is emitting logs to stdout/stderr", func() { ginkgo.It("should return at least one line of log", func() { - defer results.RecordResult(identifiers.TestLoggingIdentifier) - loggingTest(podNameSpace, podName, containerName) + for _, cut := range configData.ContainersUnderTest { + podName := cut.Oc.GetPodName() + podNamespace := cut.Oc.GetPodNamespace() + containerName := cut.Oc.GetPodContainerName() + ginkgo.By(fmt.Sprintf("Test podnamespace=%s podname=%s", + podNamespace, podName)) + defer results.RecordResult(identifiers.TestLoggingIdentifier) + loggingTest(podNamespace, podName, containerName) + } }) }) } diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 41879c06e..80ce2fab2 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -95,17 +95,14 @@ var _ = ginkgo.Describe(testSpecName, func() { // testOperatorsAreInstalledViaOLM ensures all configured operators have a proper OLM subscription. func testOperatorsAreInstalledViaOLM() { - _, operatorsInTest := getConfig() - for _, operatorInTest := range operatorsInTest { - ginkgo.Context("an operator is installed", func() { - ginkgo.When("subscriptions are polled", func() { - ginkgo.It(fmt.Sprintf("%s in namespace %s Should have a valid subscription", operatorInTest.SubscriptionName, operatorInTest.Namespace), func() { - defer results.RecordResult(identifiers.TestOperatorIsInstalledViaOLMIdentifier) - testOperatorIsInstalledViaOLM(operatorInTest.SubscriptionName, operatorInTest.Namespace) - }) - }) - }) - } + ginkgo.It("operator-olm-installed", func() { + _, operatorsInTest := getConfig() + for _, operatorInTest := range operatorsInTest { + defer results.RecordResult(identifiers.TestOperatorIsInstalledViaOLMIdentifier) + ginkgo.By(fmt.Sprintf("%s in namespace %s Should have a valid subscription", operatorInTest.SubscriptionName, operatorInTest.Namespace)) + testOperatorIsInstalledViaOLM(operatorInTest.SubscriptionName, operatorInTest.Namespace) + } + }) } // testOperatorIsInstalledViaOLM tests that an operator is installed via OLM. @@ -138,47 +135,48 @@ func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsection } func itRunsTestsOnOperator() { - operatorsToQuery, operatorsInTest := getConfig() - if len(operatorsToQuery) > 0 { - certAPIClient := api.NewHTTPClient() - for _, certified := range operatorsToQuery { - // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. - ginkgo.It(fmt.Sprintf("should eventually be verified as certified (operator %s/%s)", certified.Organization, certified.Name), func() { + ginkgo.It("operator-certification", func() { + operatorsToQuery, operatorsInTest := getConfig() + if len(operatorsToQuery) > 0 { + certAPIClient := api.NewHTTPClient() + for _, certified := range operatorsToQuery { + // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. + ginkgo.By(fmt.Sprintf("should eventually be verified as certified (operator %s/%s)", certified.Organization, certified.Name)) defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) certified := certified // pin gomega.Eventually(func() bool { isCertified := certAPIClient.IsOperatorCertified(certified.Organization, certified.Name) return isCertified }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) - }) + } } - } - gomega.Expect(operatorsInTest).ToNot(gomega.BeNil()) - for _, op := range operatorsInTest { - // TODO: Gather facts for operator - for _, testType := range op.Tests { - testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) - gomega.Expect(testFile).ToNot(gomega.BeNil()) - gomega.Expect(err).To(gomega.BeNil()) - testConfigure := testcases.ContainsConfiguredTest(testFile.OperatorTest, testType) - renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Operator, testType) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) - for _, testCase := range renderedTestCase.TestCase { - if testCase.SkipTest { - continue - } - if testCase.ExpectedType == testcases.Function { - for _, val := range testCase.ExpectedStatus { - testCase.ExpectedStatusFn(op.Name, testcases.StatusFunctionType(val)) + gomega.Expect(operatorsInTest).ToNot(gomega.BeNil()) + for _, op := range operatorsInTest { + // TODO: Gather facts for operator + for _, testType := range op.Tests { + testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) + gomega.Expect(testFile).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + testConfigure := testcases.ContainsConfiguredTest(testFile.OperatorTest, testType) + renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Operator, testType) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) + for _, testCase := range renderedTestCase.TestCase { + if testCase.SkipTest { + continue + } + if testCase.ExpectedType == testcases.Function { + for _, val := range testCase.ExpectedStatus { + testCase.ExpectedStatusFn(op.Name, testcases.StatusFunctionType(val)) + } } + name := agrName(op.Name, op.SubscriptionName, testCase.Name) + args := []interface{}{name, op.Namespace} + runTestsOnOperator(args, name, op.Namespace, testCase) } - name := agrName(op.Name, op.SubscriptionName, testCase.Name) - args := []interface{}{name, op.Namespace} - runTestsOnOperator(args, name, op.Namespace, testCase) } } - } + }) } func agrName(operatorName, subName, testName string) string { diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index dde598f4d..185ba34f5 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -55,24 +55,16 @@ import ( // var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.PlatformAlterationTestKey) { - config := common.GetTestConfiguration() - log.Infof("Test Configuration: %s", config) - - containersUnderTest := common.CreateContainersUnderTest(config) - partnerContainers := common.CreatePartnerContainers(config) - fsDiffContainer := partnerContainers[config.FsDiffMasterContainer] - log.Info(containersUnderTest) + configData := common.ConfigurationData{} + configData.SetNeedsRefresh() + ginkgo.BeforeEach(func() { + common.ReloadConfiguration(&configData) + }) ginkgo.Context("Container does not have additional packages installed", func() { // use this boolean to turn off tests that require OS packages if !common.IsMinikube() { - if fsDiffContainer != nil { - for _, containerUnderTest := range containersUnderTest { - testFsDiff(fsDiffContainer.Oc, containerUnderTest.Oc) - } - } else { - log.Warn("no fs diff container is configured, cannot run fs diff test") - } + testContainersFsDiff(&configData) } }) @@ -80,43 +72,54 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { testHugepages() if !common.IsMinikube() { - for _, containersUnderTest := range containersUnderTest { - testBootParams(common.GetContext(), containersUnderTest.Oc.GetPodName(), containersUnderTest.Oc.GetPodNamespace(), containersUnderTest.Oc) - } + testBootParams(&configData) } if !common.IsMinikube() { - for _, containersUnderTest := range containersUnderTest { - // no test identifier defined, not an official test - testSysctlConfigs(common.GetContext(), containersUnderTest.Oc.GetPodName(), containersUnderTest.Oc.GetPodNamespace()) - } + testSysctlConfigs(&configData) } } }) -// Helper to test that the PUT didn't install new packages after starting, and report through Ginkgo. -func testFsDiff(masterPodOc, targetPodOc *interactive.Oc) { - ginkgo.It(fmt.Sprintf("%s(%s) should not install new packages after starting", targetPodOc.GetPodName(), targetPodOc.GetPodContainerName()), func() { - defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) - targetPodOc.GetExpecter() - containerIDTester := containerid.NewContainerID(common.DefaultTimeout) - test, err := tnf.NewTest(targetPodOc.GetExpecter(), containerIDTester, []reel.Handler{containerIDTester}, targetPodOc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - containerID := containerIDTester.GetID() - - fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID) - test, err = tnf.NewTest(masterPodOc.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, masterPodOc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err = test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) +// testContainersFsDiff test that all CUT didn't install new packages are starting +func testContainersFsDiff(configData *common.ConfigurationData) { + ginkgo.It("platform-fsdiff", func() { + fsDiffContainer := configData.FsDiffContainer + if fsDiffContainer != nil { + for _, cut := range configData.ContainersUnderTest { + podName := cut.Oc.GetPodName() + containerName := cut.Oc.GetPodContainerName() + context := cut.Oc + ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) + testContainerFsDiff(fsDiffContainer.Oc, context) + } + } else { + log.Warn("no fs diff container is configured, cannot run fs diff test") + } }) } +// testContainerFsDiff test that the CUT didn't install new packages after starting, and report through Ginkgo. +func testContainerFsDiff(masterPodOc, targetContainerOC *interactive.Oc) { + defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) + targetContainerOC.GetExpecter() + containerIDTester := containerid.NewContainerID(common.DefaultTimeout) + test, err := tnf.NewTest(targetContainerOC.GetExpecter(), containerIDTester, []reel.Handler{containerIDTester}, targetContainerOC.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + containerID := containerIDTester.GetID() + + fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID) + test, err = tnf.NewTest(masterPodOc.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, masterPodOc.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err = test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) +} + func getMcKernelArguments(context *interactive.Context, mcName string) map[string]string { mcKernelArgumentsTester := mckernelarguments.NewMcKernelArguments(common.DefaultTimeout, mcName) test, err := tnf.NewTest(context.GetExpecter(), mcKernelArgumentsTester, []reel.Handler{mcKernelArgumentsTester}, context.GetErrorChannel()) @@ -209,40 +212,58 @@ func getSysctlConfigArgs(context *interactive.Context, nodeName string) map[stri return parseSysctlSystemOutput(sysctlAllConfigsArgs) } -func testBootParams(context *interactive.Context, podName, podNamespace string, targetPodOc *interactive.Oc) { - ginkgo.It(fmt.Sprintf("Testing boot params for the pod's node %s/%s", podNamespace, podName), func() { - defer results.RecordResult(identifiers.TestUnalteredStartupBootParamsIdentifier) - nodeName := getPodNodeName(context, podName, podNamespace) - mcName := getMcName(context, nodeName) - mcKernelArgumentsMap := getMcKernelArguments(context, mcName) - currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetPodOc) - grubKernelConfigMap := getGrubKernelArgs(context, nodeName) - - for key, mcVal := range mcKernelArgumentsMap { - if currentVal, ok := currentKernelArgsMap[key]; ok { - gomega.Expect(currentVal).To(gomega.Equal(mcVal)) - } - if grubVal, ok := grubKernelConfigMap[key]; ok { - gomega.Expect(grubVal).To(gomega.Equal(mcVal)) - } +func testBootParams(configData *common.ConfigurationData) { + ginkgo.It("platform-boot-param", func() { + context := common.GetContext() + for _, cut := range configData.ContainersUnderTest { + podName := cut.Oc.GetPodName() + podNameSpace := cut.Oc.GetPodNamespace() + targetPodOc := cut.Oc + testBootParamsHelper(context, podName, podNameSpace, targetPodOc) } }) } +func testBootParamsHelper(context *interactive.Context, podName, podNamespace string, targetPodOc *interactive.Oc) { + ginkgo.By(fmt.Sprintf("Testing boot params for the pod's node %s/%s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestUnalteredStartupBootParamsIdentifier) + nodeName := getPodNodeName(context, podName, podNamespace) + mcName := getMcName(context, nodeName) + mcKernelArgumentsMap := getMcKernelArguments(context, mcName) + currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetPodOc) + grubKernelConfigMap := getGrubKernelArgs(context, nodeName) + + for key, mcVal := range mcKernelArgumentsMap { + if currentVal, ok := currentKernelArgsMap[key]; ok { + gomega.Expect(currentVal).To(gomega.Equal(mcVal)) + } + if grubVal, ok := grubKernelConfigMap[key]; ok { + gomega.Expect(grubVal).To(gomega.Equal(mcVal)) + } + } +} -func testSysctlConfigs(context *interactive.Context, podName, podNamespace string) { - ginkgo.It(fmt.Sprintf("Testing sysctl config files for the pod's node %s/%s", podNamespace, podName), func() { - nodeName := getPodNodeName(context, podName, podNamespace) - combinedSysctlSettings := getSysctlConfigArgs(context, nodeName) - mcName := getMcName(context, nodeName) - mcKernelArgumentsMap := getMcKernelArguments(context, mcName) - - for key, sysctlConfigVal := range combinedSysctlSettings { - if mcVal, ok := mcKernelArgumentsMap[key]; ok { - gomega.Expect(mcVal).To(gomega.Equal(sysctlConfigVal)) - } +func testSysctlConfigs(configData *common.ConfigurationData) { + ginkgo.It("platform-sysctl-config", func() { + context := common.GetContext() + for _, cut := range configData.ContainersUnderTest { + podName := cut.Oc.GetPodName() + podNameSpace := cut.Oc.GetPodNamespace() + testSysctlConfigsHelper(context, podName, podNameSpace) } }) } +func testSysctlConfigsHelper(context *interactive.Context, podName, podNamespace string) { + ginkgo.By(fmt.Sprintf("Testing sysctl config files for the pod's node %s/%s", podNamespace, podName)) + nodeName := getPodNodeName(context, podName, podNamespace) + combinedSysctlSettings := getSysctlConfigArgs(context, nodeName) + mcName := getMcName(context, nodeName) + mcKernelArgumentsMap := getMcKernelArguments(context, mcName) + for key, sysctlConfigVal := range combinedSysctlSettings { + if mcVal, ok := mcKernelArgumentsMap[key]; ok { + gomega.Expect(mcVal).To(gomega.Equal(sysctlConfigVal)) + } + } +} func testTainted() { if common.IsMinikube() { diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 11c1c346a..946ea96d3 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -55,5 +55,5 @@ certifiedcontainerinfo: - name: nginx-116 # working example repository: rhel8 certifiedoperatorinfo: - - name: etcd-operator - organization: redhat-marketplace \ No newline at end of file + - name: etcd + organization: community-operators # working example \ No newline at end of file From 96b3a486063e877d9dbeaf10f81e508dcb7ea902 Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:27:37 +0300 Subject: [PATCH 010/344] update (#299) * update * changed the name * changed the name! * Corrected format Golang Co-authored-by: Shimrit peretz --- cmd/tnf/main.go | 10 ++-- pkg/tnf/handlers/handler_template/doc.tmpl | 3 +- .../handlers/handler_template/handler.tmpl | 38 ++++++++------- .../handler_template/handler_test.tmpl | 48 +++++++++---------- 4 files changed, 52 insertions(+), 47 deletions(-) diff --git a/cmd/tnf/main.go b/cmd/tnf/main.go index 888a12a7f..3b914e8bf 100644 --- a/cmd/tnf/main.go +++ b/cmd/tnf/main.go @@ -5,13 +5,15 @@ import ( "log" "os" "path" + "strings" "text/template" "github.com/spf13/cobra" ) type myHandler struct { - Handlername string + UpperHandlername string + LowerHandlername string } var ( @@ -47,7 +49,7 @@ func generateHandlerFiles(cmd *cobra.Command, args []string) error { return err } - myhandler := myHandler{Handlername: handlername} + myhandler := myHandler{LowerHandlername: handlername, UpperHandlername: strings.Title(handlername)} // pathfile this is the path of the file from template file that will creat @@ -59,14 +61,14 @@ func generateHandlerFiles(cmd *cobra.Command, args []string) error { } pathfile = path.Join(handlerDirectory, "handler_template", "handler_test.tmpl") - namefile = "" + handlername + "_test.go" + namefile = "" + myhandler.LowerHandlername + "_test.go" err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template_test.tmpl if err != nil { return err } pathfile = path.Join(handlerDirectory, "handler_template", "handler.tmpl") - namefile = "" + handlername + ".go" + namefile = "" + myhandler.LowerHandlername + ".go" err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template.tmpl if err != nil { return err diff --git a/pkg/tnf/handlers/handler_template/doc.tmpl b/pkg/tnf/handlers/handler_template/doc.tmpl index 758d74831..323a41ad2 100644 --- a/pkg/tnf/handlers/handler_template/doc.tmpl +++ b/pkg/tnf/handlers/handler_template/doc.tmpl @@ -13,4 +13,5 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package {{ .Handlername }} \ No newline at end of file +// Package {{ .LowerHandlername }} provides a test for {{ .LowerHandlername }}. +package {{ .LowerHandlername }} diff --git a/pkg/tnf/handlers/handler_template/handler.tmpl b/pkg/tnf/handlers/handler_template/handler.tmpl index 48fabf222..39f5e886c 100644 --- a/pkg/tnf/handlers/handler_template/handler.tmpl +++ b/pkg/tnf/handlers/handler_template/handler.tmpl @@ -14,16 +14,17 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package {{ .Handlername }} +package {{ .LowerHandlername }} import ( "time" + "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) -// TODO: -type {{ .Handlername }} struct { +// {{ .UpperHandlername }} provides a {{ .UpperHandlername }} test implemented using command line tool {{ .UpperHandlername }}. +type {{ .UpperHandlername }} struct { result int timeout time.Duration args []string @@ -31,50 +32,51 @@ type {{ .Handlername }} struct { } const ( - // adding special variables +// adding special variables ) -func (h *{{ .Handlername }}) Args() []string { +// Args function +func (h *{{ .UpperHandlername }}) Args() []string { return h.args } // GetIdentifier returns the tnf.Test specific identifier. -func (h *{{ .Handlername }}) GetIdentifier() { - // TODO : create identifier.{{ .Handlername}}Identifier. +func (h *{{ .UpperHandlername }}) GetIdentifier() { + // Create identifier {{ .UpperHandlername }}Identifier. } // Timeout return the timeout for the test. -func (h *{{ .Handlername }}) Timeout() time.Duration { +func (h *{{ .UpperHandlername }}) Timeout() time.Duration { return h.timeout } // Result returns the test result. -func (h *{{ .Handlername }}) Result() int { +func (h *{{ .UpperHandlername }}) Result() int { return h.result } -// ReelFirst returns a step which expects an Handlername summary for the given device. -func (h *{{ .Handlername }}) ReelFirst() *reel.Step { +// ReelFirst returns a step which expects an {{ .UpperHandlername }} summary for the given device. +func (h *{{ .UpperHandlername }}) ReelFirst() *reel.Step { return &reel.Step{ Expect: []string{""}, // TODO : pass the list of possible regex in here Timeout: h.timeout, } } -// ReelMatch parses the Handlername output and set the test result on match. -func (h *{{ .Handlername }}) ReelMatch(_, _, match string) *reel.Step { +// ReelMatch parses the {{ .UpperHandlername }} output and set the test result on match. +func (h *{{ .UpperHandlername }}) ReelMatch(_, _, match string) *reel.Step { h.result = tnf.ERROR return nil // TODO : add the matching logic in here } -// ReelTimeout does nothing; Handlername requires no explicit intervention for a timeout. -func (h *{{ .Handlername }}) ReelTimeout() *reel.Step { +// ReelTimeout does nothing, {{ .UpperHandlername }} requires no explicit intervention for a timeout. +func (h *{{ .UpperHandlername }}) ReelTimeout() *reel.Step { return nil // TODO : fill the stub } -// ReelEOF does nothing; Handlername requires no explicit intervention for EOF. -func (h *{{ .Handlername }}) ReelEOF() { +// ReelEOF does nothing, {{ .UpperHandlername }} requires no explicit intervention for EOF. +func (h *{{ .UpperHandlername }}) ReelEOF() { // TODO : fill the stub -} +} \ No newline at end of file diff --git a/pkg/tnf/handlers/handler_template/handler_test.tmpl b/pkg/tnf/handlers/handler_template/handler_test.tmpl index 6597c4a17..4f4ef5030 100644 --- a/pkg/tnf/handlers/handler_template/handler_test.tmpl +++ b/pkg/tnf/handlers/handler_template/handler_test.tmpl @@ -14,47 +14,47 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package {{ .Handlername }} +package {{ .LowerHandlername }} import ( "testing" ) const ( - // adding special variable +// adding special variable ) -// {{ .Handlername }}_Args() -func Test{{ .Handlername }}_Args(){ - // Todo : write test for function {{ .Handlername }}_Args() +// {{ .UpperHandlername }}_Args() +func Test{{ .UpperHandlername }}_Args(t *testing.T) { + // Todo: Write test for function {{ .UpperHandlername }}_Args. } -// {{ .Handlername }}_GetIdentifier -func Test{{ .Handlername }}_GetIdentifier(){ - // Todo : write test for function {{ .Handlername }}_GetIdentifier +// {{ .UpperHandlername }}_GetIdentifier +func Test{{ .UpperHandlername }}_GetIdentifier(t *testing.T) { + // Todo : write test for function {{ .UpperHandlername }}_GetIdentifier. } -// {{ .Handlername }}_ReelFirst -func Test{{ .Handlername }}_ReelFirst(){ - // Todo : write test for function {{ .Handlername }}_ReelFirst +// {{ .UpperHandlername }}_ReelFirst +func Test{{ .UpperHandlername }}_ReelFirst(t *testing.T) { + // Todo : write test for function {{ .UpperHandlername }}_ReelFirst. } -// {{ .Handlername }}_ReelEof -func Test{{ .Handlername }}_ReelEof(){ - // Todo : write test for function {{ .Handlername }}_ReelEof +// {{ .UpperHandlername }}_ReelEof +func Test{{ .UpperHandlername }}_ReelEof(t *testing.T) { + // Todo : write test for function {{ .UpperHandlername }}_ReelEof. } -// {{ .Handlername }}_ReelTimeout -func Test{{ .Handlername }}_ReelTimeout(){ - // Todo : write test for function {{ .Handlername }}_ReelTimeout +// {{ .UpperHandlername }}_ReelTimeout +func Test{{ .UpperHandlername }}_ReelTimeout(t *testing.T) { + // Todo : write test for function {{ .UpperHandlername }}_ReelTimeout. } -// {{ .Handlername }}_ReelMatch -func Test{{ .Handlername }}_ReelMatch(){ - // Todo : write test for function {{ .Handlername }}_ReelMatch +// {{ .UpperHandlername }}_ReelMatch +func Test{{ .UpperHandlername }}_ReelMatch(t *testing.T) { + // Todo : write test for function {{ .UpperHandlername }}_ReelMatch. } -// New{{ .Handlername }} -func TestNew{{ .Handlername }}(t *testing.T){ - // Todo : write test for function TestNew{{ .Handlername }} -} \ No newline at end of file +// New{{ .UpperHandlername }} +func TestNew{{ .UpperHandlername }}(t *testing.T) { + // Todo : write test for function TestNew{{ .UpperHandlername }}. +} From 6a5824e558052d26c6d60060b190dfd0eadd752e Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 17 Aug 2021 14:32:14 -0400 Subject: [PATCH 011/344] Singleton test env/config structure (#305) --- .gitignore | 3 +- pkg/config/autodiscover/autodiscover.go | 8 +- .../autodiscover/autodiscover_partner.go | 16 +- .../autodiscover/autodiscover_targets.go | 10 +- pkg/config/config.go | 164 +++++++++++++++--- ...config_test.go => config_instance_test.go} | 18 +- pkg/config/configsections/common.go | 18 +- pkg/config/configsections/container.go | 4 +- pkg/tnf/test.go | 8 + run-cnf-suites.sh | 4 +- test-network-function/accesscontrol/suite.go | 39 ++--- test-network-function/certification/suite.go | 6 +- test-network-function/common/env.go | 149 ---------------- test-network-function/generic/suite.go | 15 +- test-network-function/lifecycle/suite.go | 58 +++---- test-network-function/networking/suite.go | 30 ++-- test-network-function/observability/suite.go | 29 ++-- test-network-function/operator/suite.go | 2 +- test-network-function/platform/suite.go | 27 ++- test-network-function/suite_test.go | 2 +- 20 files changed, 283 insertions(+), 327 deletions(-) rename pkg/config/{config_test.go => config_instance_test.go} (63%) diff --git a/.gitignore b/.gitignore index 6808ee558..b8574954e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ pkg/tnf/mocks/mock_tester.go pkg/tnf/reel/mocks/mock_reel.go # NOTE: mock_expect.go is tracked, as it is generated from source externally. pkg/tnf/interactive/mocks/mock_spawner.go -temp/ \ No newline at end of file +temp/ +tnf diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 822d6736d..e4fb25ab6 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -69,7 +69,7 @@ func makeGetCommand(resourceType, labelQuery string) *exec.Cmd { } // getContainersByLabel builds `config.Container`s from containers in pods matching a label. -func getContainersByLabel(label configsections.Label) (containers []configsections.Container, err error) { +func getContainersByLabel(label configsections.Label) (containers []configsections.ContainerConfig, err error) { pods, err := GetPodsByLabel(label) if err != nil { return nil, err @@ -94,7 +94,7 @@ func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs [] // getContainerByLabel returns exactly one container with the given label. If any other number of containers is found // then an error is returned along with an empty `config.Container`. -func getContainerByLabel(label configsections.Label) (container configsections.Container, err error) { +func getContainerByLabel(label configsections.Label) (container configsections.ContainerConfig, err error) { containers, err := getContainersByLabel(label) if err != nil { return container, err @@ -106,10 +106,10 @@ func getContainerByLabel(label configsections.Label) (container configsections.C } // buildContainersFromPodResource builds `configsections.Container`s from a `PodResource` -func buildContainersFromPodResource(pr *PodResource) (containers []configsections.Container) { +func buildContainersFromPodResource(pr *PodResource) (containers []configsections.ContainerConfig) { for _, containerResource := range pr.Spec.Containers { var err error - var container configsections.Container + var container configsections.ContainerConfig container.Namespace = pr.Metadata.Namespace container.PodName = pr.Metadata.Name container.ContainerName = containerResource.Name diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index 3b8343dfb..b8ac75297 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -27,23 +27,23 @@ const ( fsDiffMasterValue = "fs_diff_master" ) -// FillTestPartner completes a `configsections.TestPartner` from the current state of the cluster, +// FindTestPartner completes a `configsections.TestPartner` from the current state of the cluster, // using labels and annotations to populate the data, if it's not fully configured -func FillTestPartner(tp *configsections.TestPartner) { - if tp.TestOrchestrator.ContainerName == "" { +func FindTestPartner(tp *configsections.TestPartner) { + if tp.TestOrchestratorID.ContainerName == "" { orchestrator, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: orchestratorValue}) if err != nil { log.Fatalf("failed to identify a single test orchestrator container: %s", err) } - tp.PartnerContainers = append(tp.PartnerContainers, orchestrator) - tp.TestOrchestrator = orchestrator.ContainerIdentifier + tp.ContainerConfigList = append(tp.ContainerConfigList, orchestrator) + tp.TestOrchestratorID = orchestrator.ContainerIdentifier } - if tp.FsDiffMasterContainer.ContainerName == "" { + if tp.FsDiffMasterContainerID.ContainerName == "" { fsDiffMasterContainer, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: fsDiffMasterValue}) if err == nil { - tp.PartnerContainers = append(tp.PartnerContainers, fsDiffMasterContainer) - tp.FsDiffMasterContainer = fsDiffMasterContainer.ContainerIdentifier + tp.ContainerConfigList = append(tp.ContainerConfigList, fsDiffMasterContainer) + tp.FsDiffMasterContainerID = fsDiffMasterContainer.ContainerIdentifier } else { log.Warnf("an error (%s) occurred when getting the FS Diff Master Container. Attempting to continue", err) } diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 52700ef7f..66df6ab2e 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -34,16 +34,16 @@ var ( podTestsAnnotationName = buildAnnotationName("host_resource_tests") ) -// FindTestTarget builds a `configsections.TestTarget` from the current state of the cluster, -// using labels and annotations to populate the data. -func FindTestTarget(labels []configsections.Label) (target configsections.TestTarget) { +// FindTestTarget finds test targets from the current state of the cluster, +// using labels and annotations, and add them to the `configsections.TestTarget` passed in. +func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget) { // find pods by label for _, l := range labels { pods, err := GetPodsByLabel(l) if err == nil { for i := range pods.Items { target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(&pods.Items[i])) - target.ContainersUnderTest = append(target.ContainersUnderTest, buildContainersFromPodResource(&pods.Items[i])...) + target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(&pods.Items[i])...) } } else { log.Warnf("failed to query by label: %v %v", l, err) @@ -65,8 +65,6 @@ func FindTestTarget(labels []configsections.Label) (target configsections.TestTa } else { log.Warnf("an error (%s) occurred when looking for operaters by label", err) } - - return target } // buildPodUnderTest builds a single `configsections.Pod` from a PodResource diff --git a/pkg/config/config.go b/pkg/config/config.go index 32faedb7e..fef871c87 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -20,25 +20,28 @@ import ( "fmt" "io/ioutil" "os" + "time" + "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/autodiscover" "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/ipaddr" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" "gopkg.in/yaml.v2" ) const ( configurationFilePathEnvironmentVariableKey = "TNF_CONFIGURATION_PATH" defaultConfigurationFilePath = "tnf_config.yml" + defaultTimeoutSeconds = 10 ) var ( - // configInstance is the singleton instance of loaded config, accessed through GetConfigInstance - configInstance configsections.TestConfiguration - // loaded tracks if the config has been loaded to prevent it being reloaded. - loaded = false - // set when an intrusive test has done something that would cause Pod/Container to be recreated - needsRefresh = false + // testEnvironment is the singleton instance of `TestEnvironment`, accessed through `GetTestEnvironment` + testEnvironment TestEnvironment ) // getConfigurationFilePathFromEnvironment returns the test configuration file. @@ -50,9 +53,82 @@ func getConfigurationFilePathFromEnvironment() string { return defaultConfigurationFilePath } +// Container is a construct which follows the Container design pattern. Essentially, a Container holds the +// pertinent information to perform a test against or using an Operating System Container. This includes facets such +// as the reference to the interactive.Oc instance, the reference to the test configuration, and the default network +// IP address. +type Container struct { + ContainerConfiguration configsections.ContainerConfig + Oc *interactive.Oc + DefaultNetworkIPAddress string + ContainerIdentifier configsections.ContainerIdentifier +} + +// DefaultTimeout for creating new interactive sessions (oc, ssh, tty) +var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second + +// Helper used to instantiate an OpenShift Client Session. +func getOcSession(pod, container, namespace string, timeout time.Duration, options ...interactive.Option) *interactive.Oc { + // Spawn an interactive OC shell using a goroutine (needed to avoid cross expect.Expecter interaction). Extract the + // Oc reference from the goroutine through a channel. Performs basic sanity checking that the Oc session is set up + // correctly. + var containerOc *interactive.Oc + ocChan := make(chan *interactive.Oc) + var chOut <-chan error + + goExpectSpawner := interactive.NewGoExpectSpawner() + var spawner interactive.Spawner = goExpectSpawner + + go func() { + oc, outCh, err := interactive.SpawnOc(&spawner, pod, container, namespace, timeout, options...) + gomega.Expect(outCh).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + ocChan <- oc + }() + + // Set up a go routine which reads from the error channel + go func() { + err := <-chOut + gomega.Expect(err).To(gomega.BeNil()) + }() + + containerOc = <-ocChan + + gomega.Expect(containerOc).ToNot(gomega.BeNil()) + + return containerOc +} + +// Extract a container IP address for a particular device. This is needed since container default network IP address +// is served by dhcp, and thus is ephemeral. +func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) string { + log.Infof("Getting IP Information for: %s(%s) in ns=%s", oc.GetPodName(), oc.GetPodContainerName(), oc.GetPodNamespace()) + ipTester := ipaddr.NewIPAddr(DefaultTimeout, dev) + test, err := tnf.NewTest(oc.GetExpecter(), ipTester, []reel.Handler{ipTester}, oc.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidateTest() + return ipTester.GetIPv4Address() +} + +// TestEnvironment includes the representation of the current state of the test targets and parters as well as the test configuration +type TestEnvironment struct { + ContainersUnderTest map[configsections.ContainerIdentifier]*Container + PartnerContainers map[configsections.ContainerIdentifier]*Container + // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from + // connectivity testing. + ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} + TestOrchestrator *Container + FsDiffMasterContainer *Container + Config configsections.TestConfiguration + // loaded tracks if the config has been loaded to prevent it being reloaded. + loaded bool + // set when an intrusive test has done something that would cause Pod/Container to be recreated + needsRefresh bool +} + // loadConfigFromFile loads a config file once. -func loadConfigFromFile(filePath string) error { - if loaded { +func (env *TestEnvironment) loadConfigFromFile(filePath string) error { + if env.loaded { return fmt.Errorf("cannot load config from file when a config is already loaded") } log.Info("Loading config from file: ", filePath) @@ -62,41 +138,77 @@ func loadConfigFromFile(filePath string) error { return err } - err = yaml.Unmarshal(contents, &configInstance) + err = yaml.Unmarshal(contents, &env.Config) if err != nil { return err } - loaded = true + env.loaded = true return nil } -// GetConfigInstance provides access to the singleton ConfigFile instance. -func GetConfigInstance() configsections.TestConfiguration { - if !loaded { +// LoadAndRefresh loads the config file if not loaded already and performs autodiscovery if needed +func (env *TestEnvironment) LoadAndRefresh() { + if !env.loaded { filePath := getConfigurationFilePathFromEnvironment() log.Debugf("GetConfigInstance before config loaded, loading from file: %s", filePath) - err := loadConfigFromFile(filePath) + err := env.loadConfigFromFile(filePath) if err != nil { log.Fatalf("unable to load configuration file: %s", err) } - autodiscover.FillTestPartner(&configInstance.TestPartner) - doAutodiscover() - } else if needsRefresh { - configInstance.TestPartner = configsections.TestPartner{} - doAutodiscover() + env.doAutodiscover() + } else if env.needsRefresh { + env.Config.Partner = configsections.TestPartner{} + env.Config.TestTarget = configsections.TestTarget{} + env.doAutodiscover() } - return configInstance } -func doAutodiscover() { +func (env *TestEnvironment) doAutodiscover() { if autodiscover.PerformAutoDiscovery() { - configInstance.TestTarget = autodiscover.FindTestTarget(configInstance.TargetPodLabels) - autodiscover.FillTestPartner(&configInstance.TestPartner) + autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget) + } + autodiscover.FindTestPartner(&env.Config.Partner) + + log.Infof("Test Configuration: %+v", *env) + + for _, cid := range env.Config.ExcludeContainersFromConnectivityTests { + env.ContainersToExcludeFromConnectivityTests[cid] = "" } - needsRefresh = false + env.ContainersUnderTest = env.createContainers(env.Config.ContainerConfigList) + env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) + env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] + env.FsDiffMasterContainer = env.PartnerContainers[env.Config.Partner.FsDiffMasterContainerID] + log.Info(env.TestOrchestrator) + log.Info(env.ContainersUnderTest) + env.needsRefresh = false +} + +// createContainers contains the general steps involved in creating "oc" sessions and other configuration. A map of the +// aggregate information is returned. +func (env *TestEnvironment) createContainers(containerDefinitions []configsections.ContainerConfig) map[configsections.ContainerIdentifier]*Container { + createdContainers := make(map[configsections.ContainerIdentifier]*Container) + for _, c := range containerDefinitions { + oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true)) + var defaultIPAddress = "UNKNOWN" + if _, ok := env.ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier]; !ok { + defaultIPAddress = getContainerDefaultNetworkIPAddress(oc, c.DefaultNetworkDevice) + } + createdContainers[c.ContainerIdentifier] = &Container{ + ContainerConfiguration: c, + Oc: oc, + DefaultNetworkIPAddress: defaultIPAddress, + ContainerIdentifier: c.ContainerIdentifier, + } + } + return createdContainers } // SetNeedsRefresh marks the config stale so that the next getInstance call will redo discovery -func SetNeedsRefresh() { - needsRefresh = true +func (env *TestEnvironment) SetNeedsRefresh() { + env.needsRefresh = true +} + +// GetTestEnvironment provides the current state of test environment +func GetTestEnvironment() *TestEnvironment { + return &testEnvironment } diff --git a/pkg/config/config_test.go b/pkg/config/config_instance_test.go similarity index 63% rename from pkg/config/config_test.go rename to pkg/config/config_instance_test.go index 0ed37790a..0d79343b8 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_instance_test.go @@ -27,16 +27,10 @@ const ( ) func TestLoadConfigFromFile(t *testing.T) { - assert.Nil(t, loadConfigFromFile(filePath)) - assert.NotNil(t, loadConfigFromFile(filePath)) // Loading when already loaded is an error case - conf := GetConfigInstance() - assert.Equal(t, conf.TestOrchestrator.Namespace, "default") - assert.Equal(t, conf.TestOrchestrator.ContainerName, "partner") - assert.Equal(t, conf.TestOrchestrator.PodName, "partner") -} - -func TestGetConfigInstance(t *testing.T) { - _ = loadConfigFromFile(filePath) - assert.NotNil(t, GetConfigInstance()) - assert.Equal(t, GetConfigInstance(), GetConfigInstance()) + env := GetTestEnvironment() + assert.Nil(t, env.loadConfigFromFile(filePath)) + assert.NotNil(t, env.loadConfigFromFile(filePath)) // Loading when already loaded is an error case + assert.Equal(t, env.Config.Partner.TestOrchestratorID.Namespace, "default") + assert.Equal(t, env.Config.Partner.TestOrchestratorID.ContainerName, "partner") + assert.Equal(t, env.Config.Partner.TestOrchestratorID.PodName, "partner") } diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 0a4d7002d..62f420e80 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -47,7 +47,7 @@ type TestConfiguration struct { // TestTarget contains k8s resources that can be targeted by tests TestTarget `yaml:"testTarget" json:"testTarget"` // TestPartner contains the helper containers that can be used to facilitate tests - TestPartner `yaml:"testPartner" json:"testPartner"` + Partner TestPartner `yaml:"testPartner" json:"testPartner"` // CertifiedContainerInfo is the list of container images to be checked for certification status. CertifiedContainerInfo []CertifiedContainerRequestInfo `yaml:"certifiedcontainerinfo,omitempty" json:"certifiedcontainerinfo,omitempty"` // CertifiedOperatorInfo is list of operator bundle names that are queried for certification status. @@ -56,20 +56,20 @@ type TestConfiguration struct { // TestPartner contains the helper containers that can be used to facilitate tests type TestPartner struct { - // PartnerContainers is the list parter containers that facilitates tests - PartnerContainers []Container `yaml:"partnerContainers" json:"partnerContainers"` - // TestOrchestrator is the id of the partner container for conducting connectivity tests - TestOrchestrator ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` - // FsDiffMasterContainer is the id of the partner container for conducting base image comparison - FsDiffMasterContainer ContainerIdentifier `yaml:"fsDiffMasterContainer" json:"fsDiffMasterContainer"` + // ContainerConfigList is the list parter containers that facilitates tests + ContainerConfigList []ContainerConfig `yaml:"partnerContainers" json:"partnerContainers"` + // TestOrchestratorID is the id of the partner container for conducting connectivity tests + TestOrchestratorID ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` + // FsDiffMasterContainerID is the id of the partner container for conducting base image comparison + FsDiffMasterContainerID ContainerIdentifier `yaml:"fsDiffMasterContainer" json:"fsDiffMasterContainer"` } // TestTarget is a collection of resources under test type TestTarget struct { // PodsUnderTest is the list of the pods that needs to be tested. Each entry is a single pod to be tested. PodsUnderTest []Pod `yaml:"podsUnderTest,omitempty" json:"podsUnderTest,omitempty"` - // ContainersUnderTest is the list of containers that needs to be tested. - ContainersUnderTest []Container `yaml:"containersUnderTest" json:"containersUnderTest"` + // ContainerConfigList is the list of containers that needs to be tested. + ContainerConfigList []ContainerConfig `yaml:"containersUnderTest" json:"containersUnderTest"` // ExcludeContainersFromConnectivityTests excludes specific containers from network connectivity tests. This is particularly useful for containers that don't have ping available. ExcludeContainersFromConnectivityTests []ContainerIdentifier `yaml:"excludeContainersFromConnectivityTests" json:"excludeContainersFromConnectivityTests"` // Operator is the list of operator objects that needs to be tested. diff --git a/pkg/config/configsections/container.go b/pkg/config/configsections/container.go index 39e1356cb..97d03c829 100644 --- a/pkg/config/configsections/container.go +++ b/pkg/config/configsections/container.go @@ -23,8 +23,8 @@ type ContainerIdentifier struct { ContainerName string `yaml:"containerName" json:"containerName"` } -// Container contains the payload of container facets. -type Container struct { +// ContainerConfig contains the payload of container facets. +type ContainerConfig struct { ContainerIdentifier `yaml:",inline"` // OpenShift Default network interface name (i.e., eth0) DefaultNetworkDevice string `yaml:"defaultNetworkDevice" json:"defaultNetworkDevice"` diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index bca9d8792..df9b76f60 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -21,6 +21,7 @@ import ( expect "github.com/google/goexpect" "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -128,6 +129,13 @@ func (t *Test) ReelEOF() { } } +// RunAndValidateTest runs the test and checks the result +func (t *Test) RunAndValidateTest() { + testResult, err := t.Run() + gomega.Expect(testResult).To(gomega.Equal(SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) +} + // NewTest creates a new Test given a chain of Handlers. func NewTest(expecter *expect.Expecter, tester Tester, chain []reel.Handler, errorChannel <-chan error, opts ...reel.Option) (*Test, error) { args := tester.Args() diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index 46b709b7a..ac9a85764 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -7,8 +7,8 @@ usage() { echo "$0 [-o OUTPUT_LOC] SUITE [... SUITE]" echo "Call the script and list the test suites to run" echo " e.g." - echo " $0 [ARGS] generic container" - echo " will run the generic and container suites" + echo " $0 [ARGS] access-control lifecycle" + echo " will run the access-control and lifecycle suites" echo "" echo "Allowed suites are listed in the README." } diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index d050f95e4..77d873e59 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -25,7 +25,7 @@ import ( ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - configpkg "github.com/test-network-function/test-network-function/pkg/config" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/clusterrolebinding" containerpkg "github.com/test-network-function/test-network-function/pkg/tnf/handlers/container" @@ -41,24 +41,21 @@ import ( var _ = ginkgo.Describe(common.AccessControlTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AccessControlTestKey) { - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) - testNamespace(&configData) + testNamespace(env) - testRoles(&configData) + testRoles(env) // Former "container" tests defer ginkgo.GinkgoRecover() // Run the tests that interact with the pods ginkgo.When("under test", func() { - conf := configpkg.GetConfigInstance() - podsUnderTest := conf.PodsUnderTest - gomega.Expect(podsUnderTest).ToNot(gomega.BeNil()) + podsUnderTest := env.Config.PodsUnderTest for _, pod := range podsUnderTest { var podFact = testcases.PodFact{Namespace: pod.Namespace, Name: pod.Name, ContainerCount: 0, HasClusterRole: false, Exists: true} // Gather facts for containers @@ -165,10 +162,10 @@ func runTestsOnPod(containerCount int, testCmd testcases.BaseTestCase, }) } -func testNamespace(configData *common.ConfigurationData) { +func testNamespace(env *config.TestEnvironment) { ginkgo.When("test deployment namespace", func() { ginkgo.It("Should not be 'default' and should not begin with 'openshift-'", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) @@ -180,15 +177,15 @@ func testNamespace(configData *common.ConfigurationData) { }) } -func testRoles(configData *common.ConfigurationData) { - testServiceAccount(configData) - testRoleBindings(configData) - testClusterRoleBindings(configData) +func testRoles(env *config.TestEnvironment) { + testServiceAccount(env) + testRoleBindings(env) + testClusterRoleBindings(env) } -func testServiceAccount(configData *common.ConfigurationData) { +func testServiceAccount(env *config.TestEnvironment) { ginkgo.It("Should have a valid ServiceAccount name", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { context := common.GetContext() podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() @@ -207,9 +204,9 @@ func testServiceAccount(configData *common.ConfigurationData) { }) } -func testRoleBindings(configData *common.ConfigurationData) { +func testRoleBindings(env *config.TestEnvironment) { ginkgo.It("Should not have RoleBinding in other namespaces", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { context := common.GetContext() podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() @@ -232,9 +229,9 @@ func testRoleBindings(configData *common.ConfigurationData) { }) } -func testClusterRoleBindings(configData *common.ConfigurationData) { +func testClusterRoleBindings(env *config.TestEnvironment) { ginkgo.It("Should not have ClusterRoleBindings", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { context := common.GetContext() podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() diff --git a/test-network-function/certification/suite.go b/test-network-function/certification/suite.go index 8e7385daf..7a3ba500a 100644 --- a/test-network-function/certification/suite.go +++ b/test-network-function/certification/suite.go @@ -52,8 +52,8 @@ func testContainerCertificationStatus() { ginkgo.When("getting certification status", func() { ginkgo.It("get certification status", func() { defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) - conf := configpkg.GetConfigInstance() - cnfsToQuery := conf.CertifiedContainerInfo + env := configpkg.GetTestEnvironment() + cnfsToQuery := env.Config.CertifiedContainerInfo if len(cnfsToQuery) > 0 { certAPIClient = api.NewHTTPClient() for _, cnf := range cnfsToQuery { @@ -73,7 +73,7 @@ func testContainerCertificationStatus() { func testOperatorCertificationStatus() { ginkgo.It("Verify operator as certified", func() { defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) - operatorsToQuery := configpkg.GetConfigInstance().CertifiedOperatorInfo + operatorsToQuery := configpkg.GetTestEnvironment().Config.CertifiedOperatorInfo if len(operatorsToQuery) > 0 { certAPIClient := api.NewHTTPClient() for _, certified := range operatorsToQuery { diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index bb62ea9c9..44fa82d54 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -23,13 +23,8 @@ import ( "time" "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/config" - "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/ipaddr" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" - "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) var ( @@ -46,94 +41,6 @@ var ( // DefaultTimeout for creating new interactive sessions (oc, ssh, tty) var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second -// ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from -// connectivity testing. -var ContainersToExcludeFromConnectivityTests = make(map[configsections.ContainerIdentifier]interface{}) - -// Helper used to instantiate an OpenShift Client Session. -func getOcSession(pod, container, namespace string, timeout time.Duration, options ...interactive.Option) *interactive.Oc { - // Spawn an interactive OC shell using a goroutine (needed to avoid cross expect.Expecter interaction). Extract the - // Oc reference from the goroutine through a channel. Performs basic sanity checking that the Oc session is set up - // correctly. - var containerOc *interactive.Oc - ocChan := make(chan *interactive.Oc) - var chOut <-chan error - - goExpectSpawner := interactive.NewGoExpectSpawner() - var spawner interactive.Spawner = goExpectSpawner - - go func() { - oc, outCh, err := interactive.SpawnOc(&spawner, pod, container, namespace, timeout, options...) - gomega.Expect(outCh).ToNot(gomega.BeNil()) - gomega.Expect(err).To(gomega.BeNil()) - ocChan <- oc - }() - - // Set up a go routine which reads from the error channel - go func() { - err := <-chOut - gomega.Expect(err).To(gomega.BeNil()) - }() - - containerOc = <-ocChan - - gomega.Expect(containerOc).ToNot(gomega.BeNil()) - - return containerOc -} - -// Container is an internal construct which follows the Container design pattern. Essentially, a Container holds the -// pertinent information to perform a test against or using an Operating System Container. This includes facets such -// as the reference to the interactive.Oc instance, the reference to the test configuration, and the default network -// IP address. -type Container struct { - ContainerConfiguration configsections.Container - Oc *interactive.Oc - DefaultNetworkIPAddress string - ContainerIdentifier configsections.ContainerIdentifier -} - -// createContainers contains the general steps involved in creating "oc" sessions and other configuration. A map of the -// aggregate information is returned. -func createContainers(containerDefinitions []configsections.Container) map[configsections.ContainerIdentifier]*Container { - createdContainers := make(map[configsections.ContainerIdentifier]*Container) - for _, c := range containerDefinitions { - oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true)) - var defaultIPAddress = "UNKNOWN" - if _, ok := ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier]; !ok { - defaultIPAddress = getContainerDefaultNetworkIPAddress(oc, c.DefaultNetworkDevice) - } - createdContainers[c.ContainerIdentifier] = &Container{ - ContainerConfiguration: c, - Oc: oc, - DefaultNetworkIPAddress: defaultIPAddress, - ContainerIdentifier: c.ContainerIdentifier, - } - } - return createdContainers -} - -// Extract a container IP address for a particular device. This is needed since container default network IP address -// is served by dhcp, and thus is ephemeral. -func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) string { - log.Infof("Getting IP Information for: %s(%s) in ns=%s", oc.GetPodName(), oc.GetPodContainerName(), oc.GetPodNamespace()) - ipTester := ipaddr.NewIPAddr(DefaultTimeout, dev) - test, err := tnf.NewTest(oc.GetExpecter(), ipTester, []reel.Handler{ipTester}, oc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - RunAndValidateTest(test) - return ipTester.GetIPv4Address() -} - -// CreateContainersUnderTest sets up the test containers. -func CreateContainersUnderTest(conf *configsections.TestConfiguration) map[configsections.ContainerIdentifier]*Container { - return createContainers(conf.ContainersUnderTest) -} - -// CreatePartnerContainers sets up the partner containers. -func CreatePartnerContainers(conf *configsections.TestConfiguration) map[configsections.ContainerIdentifier]*Container { - return createContainers(conf.PartnerContainers) -} - // GetContext spawns a new shell session and returns its context func GetContext() *interactive.Context { context, err := interactive.SpawnShell(interactive.CreateGoExpectSpawner(), DefaultTimeout, interactive.Verbose(true)) @@ -150,12 +57,6 @@ func RunAndValidateTest(test *tnf.Test) { gomega.Expect(err).To(gomega.BeNil()) } -// GetTestConfiguration returns the cnf-certification-generic-tests test configuration. -func GetTestConfiguration() *configsections.TestConfiguration { - conf := config.GetConfigInstance() - return &conf -} - // IsMinikube returns true when the env var is set, OCP only test would be skipped based on this flag func IsMinikube() bool { b, _ := strconv.ParseBool(os.Getenv("TNF_MINIKUBE_ONLY")) @@ -167,53 +68,3 @@ func NonIntrusive() bool { b, _ := strconv.ParseBool(os.Getenv("TNF_NON_INTRUSIVE_ONLY")) return b } - -// ConfigurationData is used to host test configuration -type ConfigurationData struct { - ContainersUnderTest map[configsections.ContainerIdentifier]*Container - PartnerContainers map[configsections.ContainerIdentifier]*Container - TestOrchestrator *Container - FsDiffContainer *Container - - needsRefresh bool -} - -// createContainersUnderTest sets up the test containers. -func createContainersUnderTest(conf *configsections.TestConfiguration) map[configsections.ContainerIdentifier]*Container { - return createContainers(conf.ContainersUnderTest) -} - -// createPartnerContainers sets up the partner containers. -func createPartnerContainers(conf *configsections.TestConfiguration) map[configsections.ContainerIdentifier]*Container { - return createContainers(conf.PartnerContainers) -} - -// Loadconfiguration the configuration into ConfigurationData -func Loadconfiguration(configData *ConfigurationData) { - conf := GetTestConfiguration() - log.Infof("Test Configuration: %s", conf) - - for _, cid := range conf.ExcludeContainersFromConnectivityTests { - ContainersToExcludeFromConnectivityTests[cid] = "" - } - configData.ContainersUnderTest = createContainersUnderTest(conf) - configData.PartnerContainers = createPartnerContainers(conf) - configData.TestOrchestrator = configData.PartnerContainers[conf.TestOrchestrator] - configData.FsDiffContainer = configData.PartnerContainers[conf.FsDiffMasterContainer] - log.Info(configData.TestOrchestrator) - log.Info(configData.ContainersUnderTest) -} - -// ReloadConfiguration force the autodiscovery to run again -func ReloadConfiguration(configData *ConfigurationData) { - if configData.needsRefresh { - config.SetNeedsRefresh() - Loadconfiguration(configData) - } - configData.needsRefresh = false -} - -// SetNeedsRefresh indicate the config should be reloaded after this test -func (configData *ConfigurationData) SetNeedsRefresh() { - configData.needsRefresh = true -} diff --git a/test-network-function/generic/suite.go b/test-network-function/generic/suite.go index b117e9dae..1d3a1985d 100644 --- a/test-network-function/generic/suite.go +++ b/test-network-function/generic/suite.go @@ -19,6 +19,7 @@ package generic import ( "fmt" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" "github.com/test-network-function/test-network-function/pkg/tnf/reel" @@ -42,28 +43,26 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(testsKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testsKey) { - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) - testIsRedHatRelease(&configData) + testIsRedHatRelease(env) } }) // testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. -func testIsRedHatRelease(configData *common.ConfigurationData) { +func testIsRedHatRelease(env *config.TestEnvironment) { ginkgo.It("Should report a proper Red Hat version", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { testContainerIsRedHatRelease(cut) } - testContainerIsRedHatRelease(configData.TestOrchestrator) }) } // testContainerIsRedHatRelease tests whether the container attached to oc is Red Hat based. -func testContainerIsRedHatRelease(cut *common.Container) { +func testContainerIsRedHatRelease(cut *config.Container) { podName := cut.Oc.GetPodName() containerName := cut.Oc.GetPodContainerName() context := cut.Oc diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 67a1c2fc0..535ece422 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -85,28 +86,27 @@ var drainTimeout = time.Duration(drainTimeoutMinutes) * time.Minute // All actual test code belongs below here. Utilities belong above. // var _ = ginkgo.Describe(common.LifecycleTestKey, func() { - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.LifecycleTestKey) { - testNodeSelector(&configData) + testNodeSelector(env) - testGracePeriod(&configData) + testGracePeriod(env) - testShutdown(&configData) + testShutdown(env) - testPodAntiAffinity(&configData) + testPodAntiAffinity(env) if !common.NonIntrusive() { - testPodsRecreation(&configData) + testPodsRecreation(env) - testScaling(&configData) + testScaling(env) } - testOwner(&configData) + testOwner(env) } }) @@ -118,7 +118,7 @@ func waitForAllDeploymentsReady(namespace string, timeout, pollingPeriod time.Du } // restoreDeployments is the last attempt to restore the original test deployments' replicaCount -func restoreDeployments(configData *common.ConfigurationData, nsDeployments *map[string]dp.DeploymentMap) { +func restoreDeployments(env *config.TestEnvironment, nsDeployments *map[string]dp.DeploymentMap) { for namespace, originalDeployments := range *nsDeployments { // For each deployment in the namespace, get the current replicas and compare. deployments, notReadyDeployments := getDeployments(namespace) @@ -138,7 +138,7 @@ func restoreDeployments(configData *common.ConfigurationData, nsDeployments *map // Try to scale to the original deployment's replicaCount. runScalingTest(namespace, originalDeploymentName, originalDeployment.Replicas) - configData.SetNeedsRefresh() + env.SetNeedsRefresh() } } } @@ -168,17 +168,17 @@ func runScalingTest(namespace, deploymentName string, replicaCount int) { waitForAllDeploymentsReady(namespace, scalingTimeout, scalingPollingPeriod) } -func testScaling(configData *common.ConfigurationData) { +func testScaling(env *config.TestEnvironment) { ginkgo.It("Testing deployment scaling", func() { defer results.RecordResult(identifiers.TestScalingIdentifier) namespaceDeploymentsBackup := make(map[string]dp.DeploymentMap) - defer restoreDeployments(configData, &namespaceDeploymentsBackup) + defer restoreDeployments(env, &namespaceDeploymentsBackup) // Map to register the deployments that have been already tested deploymentNames := make(map[string]bool) - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { namespace := cut.Oc.GetPodNamespace() // Get deployment name and check whether it was already tested. @@ -204,7 +204,7 @@ func testScaling(configData *common.ConfigurationData) { runScalingTest(namespace, deploymentName, replicaCount) // Ensure next tests/test suites receive a refreshed config. - configData.SetNeedsRefresh() + env.SetNeedsRefresh() // Set this deployment as tested deploymentNames[deploymentName] = true @@ -212,9 +212,9 @@ func testScaling(configData *common.ConfigurationData) { }) } -func testNodeSelector(configData *common.ConfigurationData) { +func testNodeSelector(env *config.TestEnvironment) { ginkgo.It("Testing pod nodeSelector", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() ginkgo.By(fmt.Sprintf("Testing pod nodeSelector %s/%s", cut.Oc.GetPodNamespace(), podName)) @@ -234,10 +234,10 @@ func testNodeSelector(configData *common.ConfigurationData) { }) } -func testGracePeriod(configData *common.ConfigurationData) { +func testGracePeriod(env *config.TestEnvironment) { ginkgo.When("Test terminationGracePeriod ", func() { ginkgo.It("Testing pod terminationGracePeriod", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { context := common.GetContext() podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() @@ -261,10 +261,10 @@ func testGracePeriod(configData *common.ConfigurationData) { }) } -func testShutdown(configData *common.ConfigurationData) { +func testShutdown(env *config.TestEnvironment) { ginkgo.When("Testing PUTs are configured with pre-stop lifecycle", func() { ginkgo.It("should have pre-stop configured", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNamespace := cut.Oc.GetPodNamespace() ginkgo.By(fmt.Sprintf("should have pre-stop configured %s/%s", podNamespace, podName)) @@ -298,13 +298,13 @@ func shutdownTest(podNamespace, podName string) { gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) } -func testPodsRecreation(configData *common.ConfigurationData) { +func testPodsRecreation(env *config.TestEnvironment) { var deployments dp.DeploymentMap var notReadyDeployments []string var nodesSorted []node // A slice version of nodes sorted by number of deployments descending ginkgo.It("Testing node draining effect of deployment", func() { - configData.SetNeedsRefresh() - for _, cut := range configData.ContainersUnderTest { + env.SetNeedsRefresh() + for _, cut := range env.ContainersUnderTest { namespace := cut.Oc.GetPodNamespace() ginkgo.By(fmt.Sprintf("test deployment in namespace %s", namespace)) deployments, notReadyDeployments = getDeployments(namespace) @@ -419,11 +419,11 @@ func uncordonNode(node string) { } // Pod antiaffinity test for all deployments -func testPodAntiAffinity(configData *common.ConfigurationData) { +func testPodAntiAffinity(env *config.TestEnvironment) { var deployments dp.DeploymentMap ginkgo.When("CNF is designed in high availability mode ", func() { ginkgo.It("Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podNamespace := cut.Oc.GetPodNamespace() defer results.RecordResult(identifiers.TestPodHighAvailabilityBestPractices) deployments, _ = getDeployments(podNamespace) @@ -477,10 +477,10 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) } -func testOwner(configData *common.ConfigurationData) { +func testOwner(env *config.TestEnvironment) { ginkgo.When("Testing owners of CNF pod", func() { ginkgo.It("Should be only ReplicaSet", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podNamespace := cut.Oc.GetPodNamespace() podName := cut.Oc.GetPodName() ginkgo.By(fmt.Sprintf("Should be ReplicaSet %s %s", podNamespace, podName)) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 74f6c3100..8c2c5252a 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -19,6 +19,7 @@ package networking import ( "fmt" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -46,36 +47,35 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(common.NetworkingTestKey, func() { - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.NetworkingTestKey) { ginkgo.Context("Both Pods are on the Default network", func() { // for each container under test, ensure bidirectional ICMP traffic between the container and the orchestrator. - testDefaultNetworkConnectivity(&configData, defaultNumPings) + testDefaultNetworkConnectivity(env, defaultNumPings) }) ginkgo.Context("Both Pods are connected via a Multus Overlay Network", func() { // Unidirectional test; for each container under test, attempt to ping the target Multus IP addresses. - testMultusNetworkConnectivity(&configData, defaultNumPings) + testMultusNetworkConnectivity(env, defaultNumPings) }) ginkgo.Context("Should not have type of nodePort", func() { - testNodePort(&configData) + testNodePort(env) }) } }) -func testDefaultNetworkConnectivity(configData *common.ConfigurationData, count int) { +func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { ginkgo.It("should reply to ping", func() { - for _, cut := range configData.ContainersUnderTest { - if _, ok := common.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { + for _, cut := range env.ContainersUnderTest { + if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue } context := cut.Oc - testOrchestrator := configData.TestOrchestrator + testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), cut.DefaultNetworkIPAddress)) @@ -90,12 +90,12 @@ func testDefaultNetworkConnectivity(configData *common.ConfigurationData, count }) } -func testMultusNetworkConnectivity(configData *common.ConfigurationData, count int) { +func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { ginkgo.It("should reply to ping", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { - testOrchestrator := configData.TestOrchestrator + testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), cut.DefaultNetworkIPAddress)) @@ -119,9 +119,9 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count gomega.Expect(errors).To(gomega.BeZero()) } -func testNodePort(configData *common.ConfigurationData) { +func testNodePort(env *config.TestEnvironment) { ginkgo.It("Should not have services of type NodePort", func() { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) context := common.GetContext() podNamespace := cut.Oc.GetPodNamespace() diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 362f5d347..330b855d8 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -23,6 +23,8 @@ import ( "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" + "github.com/test-network-function/test-network-function/pkg/config" + "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -44,36 +46,31 @@ var ( var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.ObservabilityTestKey) { - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) - testLogging(&configData) + testLogging(env) } }) -func testLogging(configData *common.ConfigurationData) { +func testLogging(env *config.TestEnvironment) { ginkgo.When("Testing PUT is emitting logs to stdout/stderr", func() { ginkgo.It("should return at least one line of log", func() { - for _, cut := range configData.ContainersUnderTest { - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() - containerName := cut.Oc.GetPodContainerName() - ginkgo.By(fmt.Sprintf("Test podnamespace=%s podname=%s", - podNamespace, podName)) + for _, cut := range env.ContainersUnderTest { + ginkgo.By(fmt.Sprintf("Test container: %+v", cut.ContainerIdentifier)) defer results.RecordResult(identifiers.TestLoggingIdentifier) - loggingTest(podNamespace, podName, containerName) + loggingTest(cut.ContainerIdentifier) } }) }) } -func loggingTest(podNamespace, podName, containerName string) { +func loggingTest(c configsections.ContainerIdentifier) { context := common.GetContext() values := make(map[string]interface{}) - values["POD_NAMESPACE"] = podNamespace - values["POD_NAME"] = podName - values["CONTAINER_NAME"] = containerName + values["POD_NAMESPACE"] = c.Namespace + values["POD_NAME"] = c.PodName + values["CONTAINER_NAME"] = c.ContainerName test, handlers, result, err := generic.NewGenericFromMap(relativeLoggingTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 80ce2fab2..7502b46cd 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -128,7 +128,7 @@ func testOperatorIsInstalledViaOLM(subscriptionName, subscriptionNamespace strin } func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsections.Operator) { - conf := config.GetConfigInstance() + conf := config.GetTestEnvironment().Config operatorsToQuery := conf.CertifiedOperatorInfo operatorsInTest := conf.Operators return operatorsToQuery, operatorsInTest diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 185ba34f5..411ff7e5a 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -22,6 +22,7 @@ import ( "regexp" "strings" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -55,16 +56,14 @@ import ( // var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.PlatformAlterationTestKey) { - - configData := common.ConfigurationData{} - configData.SetNeedsRefresh() + env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { - common.ReloadConfiguration(&configData) + env.LoadAndRefresh() }) ginkgo.Context("Container does not have additional packages installed", func() { // use this boolean to turn off tests that require OS packages if !common.IsMinikube() { - testContainersFsDiff(&configData) + testContainersFsDiff(env) } }) @@ -72,22 +71,22 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { testHugepages() if !common.IsMinikube() { - testBootParams(&configData) + testBootParams(env) } if !common.IsMinikube() { - testSysctlConfigs(&configData) + testSysctlConfigs(env) } } }) // testContainersFsDiff test that all CUT didn't install new packages are starting -func testContainersFsDiff(configData *common.ConfigurationData) { +func testContainersFsDiff(env *config.TestEnvironment) { ginkgo.It("platform-fsdiff", func() { - fsDiffContainer := configData.FsDiffContainer + fsDiffContainer := env.FsDiffMasterContainer if fsDiffContainer != nil { - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() containerName := cut.Oc.GetPodContainerName() context := cut.Oc @@ -212,10 +211,10 @@ func getSysctlConfigArgs(context *interactive.Context, nodeName string) map[stri return parseSysctlSystemOutput(sysctlAllConfigsArgs) } -func testBootParams(configData *common.ConfigurationData) { +func testBootParams(env *config.TestEnvironment) { ginkgo.It("platform-boot-param", func() { context := common.GetContext() - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNameSpace := cut.Oc.GetPodNamespace() targetPodOc := cut.Oc @@ -242,10 +241,10 @@ func testBootParamsHelper(context *interactive.Context, podName, podNamespace st } } -func testSysctlConfigs(configData *common.ConfigurationData) { +func testSysctlConfigs(env *config.TestEnvironment) { ginkgo.It("platform-sysctl-config", func() { context := common.GetContext() - for _, cut := range configData.ContainersUnderTest { + for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNameSpace := cut.Oc.GetPodNamespace() testSysctlConfigsHelper(context, podName, podNameSpace) diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 661faa9b3..0aea6e1bf 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -184,7 +184,7 @@ func appendCNFFeatureValidationReportResults(junitPath *string, junitMap map[str // marshalConfigurations creates a byte stream representation of the test configurations. In the event of an error, // this method fatally fails. func marshalConfigurations() []byte { - configurations, err := j.Marshal(config.GetConfigInstance()) + configurations, err := j.Marshal(config.GetTestEnvironment().Config) if err != nil { log.Fatalf("error converting configurations to JSON: %v", err) } From f00588fb70b61c2f87544bf9df7981a77f1f598d Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 18 Aug 2021 13:51:18 -0400 Subject: [PATCH 012/344] Update pod level test cases to loop through pod list (#302) --- .../autodiscover/autodiscover_targets.go | 37 ++-- pkg/config/autodiscover/pod_info.go | 3 +- pkg/config/config.go | 2 + pkg/config/configsections/pod.go | 6 + pkg/tnf/testcases/base.go | 20 +- test-network-function/accesscontrol/suite.go | 181 +++++++----------- test-network-function/lifecycle/suite.go | 59 +++--- test-network-function/networking/suite.go | 4 +- test-network-function/platform/suite.go | 24 +-- 9 files changed, 158 insertions(+), 178 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 66df6ab2e..b5f6de356 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -23,7 +23,6 @@ import ( ) const ( - configuredTestFile = "testconfigure.yml" operatorLabelName = "operator" skipConnectivityTestsLabel = "skip_connectivity_tests" ) @@ -68,37 +67,23 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa } // buildPodUnderTest builds a single `configsections.Pod` from a PodResource -func buildPodUnderTest(pr *PodResource) (cnf configsections.Pod) { +func buildPodUnderTest(pr *PodResource) (podUnderTest configsections.Pod) { var err error - cnf.Namespace = pr.Metadata.Namespace - cnf.Name = pr.Metadata.Name - + podUnderTest.Namespace = pr.Metadata.Namespace + podUnderTest.Name = pr.Metadata.Name + podUnderTest.ServiceAccount = pr.Spec.ServiceAccount + podUnderTest.ContainerCount = len(pr.Spec.Containers) var tests []string err = pr.GetAnnotationValue(podTestsAnnotationName, &tests) if err != nil { - log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", cnf.Namespace, cnf.Name, err) - cnf.Tests = getConfiguredPodTests() + log.Warnf("unable to extract tests from annotation on '%s/%s' (error: %s). Attempting to fallback to all tests", podUnderTest.Namespace, podUnderTest.Name, err) + podUnderTest.Tests = testcases.GetConfiguredPodTests() } else { - cnf.Tests = tests + podUnderTest.Tests = tests } return } -// getConfiguredPodTests loads the `configuredTestFile` and extracts -// the names of test groups from it. -func getConfiguredPodTests() (cnfTests []string) { - configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) - if err != nil { - log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) - return []string{} - } - for _, configuredTest := range configuredTests.CnfTest { - cnfTests = append(cnfTests, configuredTest.Name) - } - log.WithField("cnfTests", cnfTests).Infof("got all tests from %s.", configuredTestFile) - return cnfTests -} - // buildOperatorFromCSVResource builds a single `configsections.Operator` from a CSVResource func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) { var err error @@ -127,14 +112,14 @@ func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) // getConfiguredOperatorTests loads the `configuredTestFile` used by the `operator` specs and extracts // the names of test groups from it. func getConfiguredOperatorTests() (opTests []string) { - configuredTests, err := testcases.LoadConfiguredTestFile(configuredTestFile) + configuredTests, err := testcases.LoadConfiguredTestFile(testcases.ConfiguredTestFile) if err != nil { - log.Errorf("failed to load %s, continuing with no tests", configuredTestFile) + log.Errorf("failed to load %s, continuing with no tests", testcases.ConfiguredTestFile) return []string{} } for _, configuredTest := range configuredTests.OperatorTest { opTests = append(opTests, configuredTest.Name) } - log.WithField("opTests", opTests).Infof("got all tests from %s.", configuredTestFile) + log.WithField("opTests", opTests).Infof("got all tests from %s.", testcases.ConfiguredTestFile) return opTests } diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 30d7260cc..ba01dcedd 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -50,7 +50,8 @@ type PodResource struct { Annotations map[string]string `json:"annotations"` } `json:"metadata"` Spec struct { - Containers []struct { + ServiceAccount string `json:"serviceaccountname"` + Containers []struct { Name string `json:"name"` } `json:"containers"` } `json:"spec"` diff --git a/pkg/config/config.go b/pkg/config/config.go index fef871c87..b6e770afc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -114,6 +114,7 @@ func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) string type TestEnvironment struct { ContainersUnderTest map[configsections.ContainerIdentifier]*Container PartnerContainers map[configsections.ContainerIdentifier]*Container + PodsUnderTest []configsections.Pod // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -175,6 +176,7 @@ func (env *TestEnvironment) doAutodiscover() { env.ContainersToExcludeFromConnectivityTests[cid] = "" } env.ContainersUnderTest = env.createContainers(env.Config.ContainerConfigList) + env.PodsUnderTest = env.Config.PodsUnderTest env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.FsDiffMasterContainer = env.PartnerContainers[env.Config.Partner.FsDiffMasterContainerID] diff --git a/pkg/config/configsections/pod.go b/pkg/config/configsections/pod.go index 3b74956af..992e08e5c 100644 --- a/pkg/config/configsections/pod.go +++ b/pkg/config/configsections/pod.go @@ -24,6 +24,12 @@ type Pod struct { // Namespace where the Pod is deployed Namespace string `yaml:"namespace" json:"namespace"` + // ServiceAccount name used by the pod + ServiceAccount string `yaml:"serviceaccount" json:"serviceaccount"` + + // ContainerCount is the count of containers inside the pod + ContainerCount int `yaml:"containercount" json:"containercount"` + // Tests this is list of test that need to run against the Pod. Tests []string `yaml:"tests" json:"tests"` } diff --git a/pkg/tnf/testcases/base.go b/pkg/tnf/testcases/base.go index 2b9d58c99..693c6c95d 100644 --- a/pkg/tnf/testcases/base.go +++ b/pkg/tnf/testcases/base.go @@ -23,6 +23,7 @@ import ( "regexp" "strings" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf/testcases/data/cnf" "github.com/test-network-function/test-network-function/pkg/tnf/testcases/data/operator" "gopkg.in/yaml.v2" @@ -34,6 +35,8 @@ type StatusFunctionType string const ( // ServiceAccountFn function name to be called to replace expected status ServiceAccountFn StatusFunctionType = "FN_SERVICE_ACCOUNT_NAME" + // ConfiguredTestFile the name for the default "container" test list file + ConfiguredTestFile = "testconfigure.yml" ) // TestResultType Defines Test Result Type @@ -114,8 +117,6 @@ type PodFact struct { Namespace string // ServiceAccount name used by the pod ServiceAccount string - // HasClusterRole if pod has cluster role - HasClusterRole bool // ContainerCount is the count of containers inside the pod ContainerCount int // Exists if the pod is found in the cluster @@ -345,3 +346,18 @@ func IsInFocus(focus []string, desc string) bool { } return matchesFocus } + +// GetConfiguredPodTests loads the `configuredTestFile` and extracts +// the names of test groups from it. +func GetConfiguredPodTests() (cnfTests []string) { + configuredTests, err := LoadConfiguredTestFile(ConfiguredTestFile) + if err != nil { + log.Errorf("failed to load %s, continuing with no tests", ConfiguredTestFile) + return []string{} + } + for _, configuredTest := range configuredTests.CnfTest { + cnfTests = append(cnfTests, configuredTest.Name) + } + log.WithField("cnfTests", cnfTests).Infof("got all tests from %s.", ConfiguredTestFile) + return cnfTests +} diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 77d873e59..208e905fa 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -18,7 +18,6 @@ package accesscontrol import ( "fmt" - "strconv" "strings" "github.com/onsi/ginkgo" @@ -31,7 +30,6 @@ import ( containerpkg "github.com/test-network-function/test-network-function/pkg/tnf/handlers/container" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/rolebinding" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/serviceaccount" - "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -50,67 +48,22 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { testRoles(env) - // Former "container" tests defer ginkgo.GinkgoRecover() // Run the tests that interact with the pods ginkgo.When("under test", func() { - podsUnderTest := env.Config.PodsUnderTest - for _, pod := range podsUnderTest { - var podFact = testcases.PodFact{Namespace: pod.Namespace, Name: pod.Name, ContainerCount: 0, HasClusterRole: false, Exists: true} - // Gather facts for containers - podFacts, err := testcases.LoadCnfTestCaseSpecs(testcases.GatherFacts) + var allTests []string = testcases.GetConfiguredPodTests() + for _, testType := range allTests { + testFile, err := testcases.LoadConfiguredTestFile(common.ConfiguredTestFile) + gomega.Expect(testFile).ToNot(gomega.BeNil()) gomega.Expect(err).To(gomega.BeNil()) - context := common.GetContext() - // Collect container facts - for _, factsTest := range podFacts.TestCase { - args := strings.Split(fmt.Sprintf(factsTest.Command, pod.Name, pod.Namespace), " ") - podTest := containerpkg.NewPod(args, pod.Name, pod.Namespace, factsTest.ExpectedStatus, factsTest.ResultType, factsTest.Action, common.DefaultTimeout) - test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - _, err = test.Run() - gomega.Expect(err).To(gomega.BeNil()) - if factsTest.Name == string(testcases.ContainerCount) { - podFact.ContainerCount, _ = strconv.Atoi(podTest.Facts()) - } else if factsTest.Name == string(testcases.ServiceAccountName) { - podFact.ServiceAccount = podTest.Facts() - } else if factsTest.Name == string(testcases.Name) { - podFact.Name = podTest.Facts() - gomega.Expect(podFact.Name).To(gomega.Equal(pod.Name)) - if strings.Compare(podFact.Name, pod.Name) > 0 { - podFact.Exists = true - } - } - } - // loop through various cnfs test - if !podFact.Exists { - ginkgo.It(fmt.Sprintf("is running test pod exists : %s/%s for test command : %s", podFact.Namespace, podFact.Name, "POD EXISTS"), func() { - gomega.Expect(podFact.Exists).To(gomega.BeTrue()) - }) - continue - } - for _, testType := range pod.Tests { - testFile, err := testcases.LoadConfiguredTestFile(common.ConfiguredTestFile) - gomega.Expect(testFile).ToNot(gomega.BeNil()) - gomega.Expect(err).To(gomega.BeNil()) - testConfigure := testcases.ContainsConfiguredTest(testFile.CnfTest, testType) - renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Cnf, testType) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) - for _, testCase := range renderedTestCase.TestCase { - if !testCase.SkipTest { - if testCase.ExpectedType == testcases.Function { - for _, val := range testCase.ExpectedStatus { - testCase.ExpectedStatusFn(pod.Name, testcases.StatusFunctionType(val)) - } - } - if testCase.Loop > 0 { - runTestsOnPod(podFact.ContainerCount, testCase, testType, podFact, context) - } else { - runTestsOnPod(testCase.Loop, testCase, testType, podFact, context) - } - } + testConfigure := testcases.ContainsConfiguredTest(testFile.CnfTest, testType) + renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Cnf, testType) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) + for _, testCase := range renderedTestCase.TestCase { + if !testCase.SkipTest { + runTestOnPods(env, testCase, testType) } } } @@ -118,46 +71,61 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { } }) -//nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. -func runTestsOnPod(containerCount int, testCmd testcases.BaseTestCase, - testType string, facts testcases.PodFact, context *interactive.Context) { - ginkgo.It(fmt.Sprintf("is running test for : %s/%s for test command : %s", facts.Namespace, facts.Name, testCmd.Name), func() { - defer results.RecordResult(identifiers.TestHostResourceIdentifier) - containerCount := containerCount - testType := testType - facts := facts - testCmd := testCmd - var args []interface{} - if testType == testcases.PrivilegedRoles { - args = []interface{}{facts.Namespace, facts.Namespace, facts.ServiceAccount} - } else { - args = []interface{}{facts.Name, facts.Namespace} - } - if containerCount > 0 { - count := 0 - for count < containerCount { - argsCount := append(args, count) - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, argsCount...), " ") - cnfInTest := containerpkg.NewPod(cmdArgs, facts.Name, facts.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) - gomega.Expect(cnfInTest).ToNot(gomega.BeNil()) - test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) +//nolint:gocritic,funlen // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. +func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, testType string) { + ginkgo.It(fmt.Sprintf("Running Pod test : %s", testCmd.Name), func() { + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) + if testCmd.ExpectedType == testcases.Function { + for _, val := range testCmd.ExpectedStatus { + testCmd.ExpectedStatusFn(podName, testcases.StatusFunctionType(val)) + } + } + defer results.RecordResult(identifiers.TestHostResourceIdentifier) + testType := testType + testCmd := testCmd + var args []interface{} + if testType == testcases.PrivilegedRoles { + args = []interface{}{podUnderTest.Namespace, podUnderTest.Namespace, podUnderTest.ServiceAccount} + } else { + args = []interface{}{podUnderTest.Name, podUnderTest.Namespace} + } + var count int + if testCmd.Loop > 0 { + count = podUnderTest.ContainerCount + } else { + count = testCmd.Loop + } + + if count > 0 { + count := 0 + for count < podUnderTest.ContainerCount { + argsCount := append(args, count) + cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, argsCount...), " ") + cnfInTest := containerpkg.NewPod(cmdArgs, podUnderTest.Name, podUnderTest.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) + gomega.Expect(cnfInTest).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + count++ + } + } else { + cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") + podTest := containerpkg.NewPod(cmdArgs, podUnderTest.Name, podUnderTest.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) + gomega.Expect(podTest).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - count++ } - } else { - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") - podTest := containerpkg.NewPod(cmdArgs, facts.Name, facts.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) - gomega.Expect(podTest).ToNot(gomega.BeNil()) - test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) } }) } @@ -165,9 +133,9 @@ func runTestsOnPod(containerCount int, testCmd testcases.BaseTestCase, func testNamespace(env *config.TestEnvironment) { ginkgo.When("test deployment namespace", func() { ginkgo.It("Should not be 'default' and should not begin with 'openshift-'", func() { - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) defer results.RecordResult(identifiers.TestNamespaceBestPracticesIdentifier) gomega.Expect(podNamespace).To(gomega.Not(gomega.Equal("default"))) @@ -185,10 +153,10 @@ func testRoles(env *config.TestEnvironment) { func testServiceAccount(env *config.TestEnvironment) { ginkgo.It("Should have a valid ServiceAccount name", func() { - for _, cut := range env.ContainersUnderTest { + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace context := common.GetContext() - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() ginkgo.By(fmt.Sprintf("Testing pod service account %s %s", podNamespace, podName)) defer results.RecordResult(identifiers.TestPodServiceAccountBestPracticesIdentifier) tester := serviceaccount.NewServiceAccount(common.DefaultTimeout, podName, podNamespace) @@ -198,7 +166,6 @@ func testServiceAccount(env *config.TestEnvironment) { gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) serviceAccountName := tester.GetServiceAccountName() - cut.Oc.SetServiceAccountName(serviceAccountName) gomega.Expect(serviceAccountName).ToNot(gomega.BeEmpty()) } }) @@ -206,11 +173,11 @@ func testServiceAccount(env *config.TestEnvironment) { func testRoleBindings(env *config.TestEnvironment) { ginkgo.It("Should not have RoleBinding in other namespaces", func() { - for _, cut := range env.ContainersUnderTest { + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + serviceAccountName := podUnderTest.ServiceAccount context := common.GetContext() - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() - serviceAccountName := cut.Oc.GetServiceAccountName() defer results.RecordResult(identifiers.TestPodRoleBindingsBestPracticesIdentifier) ginkgo.By(fmt.Sprintf("Testing role bidning %s %s", podNamespace, podName)) if serviceAccountName == "" { @@ -231,11 +198,11 @@ func testRoleBindings(env *config.TestEnvironment) { func testClusterRoleBindings(env *config.TestEnvironment) { ginkgo.It("Should not have ClusterRoleBindings", func() { - for _, cut := range env.ContainersUnderTest { + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + serviceAccountName := podUnderTest.ServiceAccount context := common.GetContext() - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() - serviceAccountName := cut.Oc.GetServiceAccountName() defer results.RecordResult(identifiers.TestPodClusterRoleBindingsBestPracticesIdentifier) ginkgo.By(fmt.Sprintf("Testing cluster role bidning %s %s", podNamespace, podName)) if serviceAccountName == "" { diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 535ece422..2318ea11e 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -178,14 +178,16 @@ func testScaling(env *config.TestEnvironment) { // Map to register the deployments that have been already tested deploymentNames := make(map[string]bool) - for _, cut := range env.ContainersUnderTest { - namespace := cut.Oc.GetPodNamespace() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + namespace := podUnderTest.Namespace // Get deployment name and check whether it was already tested. // ToDo: Proper way (helper/handler) to do this. - podNameParts := strings.Split(cut.Oc.GetPodName(), "-") + podNameParts := strings.Split(podName, "-") deploymentName := podNameParts[0] - + msg := fmt.Sprintf("Testing deployment=%s, namespace=%s pod name=%s", deploymentName, namespace, podName) + log.Info(msg) if _, alreadyTested := deploymentNames[deploymentName]; alreadyTested { continue } @@ -214,14 +216,15 @@ func testScaling(env *config.TestEnvironment) { func testNodeSelector(env *config.TestEnvironment) { ginkgo.It("Testing pod nodeSelector", func() { - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() - ginkgo.By(fmt.Sprintf("Testing pod nodeSelector %s/%s", cut.Oc.GetPodNamespace(), podName)) + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("Testing pod nodeSelector %s/%s", podNamespace, podName)) defer results.RecordResult(identifiers.TestPodNodeSelectorAndAffinityBestPractices) infoWriter := tnf.CreateTestExtraInfoWriter() tester := nodeselector.NewNodeSelector(common.DefaultTimeout, podName, podNamespace) - test, err := tnf.NewTest(cut.Oc.GetExpecter(), tester, []reel.Handler{tester}, cut.Oc.GetErrorChannel()) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(err).To(gomega.BeNil()) @@ -237,10 +240,10 @@ func testNodeSelector(env *config.TestEnvironment) { func testGracePeriod(env *config.TestEnvironment) { ginkgo.When("Test terminationGracePeriod ", func() { ginkgo.It("Testing pod terminationGracePeriod", func() { - for _, cut := range env.ContainersUnderTest { - context := common.GetContext() - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing pod terminationGracePeriod %s %s", podNamespace, podName)) defer results.RecordResult(identifiers.TestNonDefaultGracePeriodIdentifier) infoWriter := tnf.CreateTestExtraInfoWriter() @@ -264,9 +267,9 @@ func testGracePeriod(env *config.TestEnvironment) { func testShutdown(env *config.TestEnvironment) { ginkgo.When("Testing PUTs are configured with pre-stop lifecycle", func() { ginkgo.It("should have pre-stop configured", func() { - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - podNamespace := cut.Oc.GetPodNamespace() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("should have pre-stop configured %s/%s", podNamespace, podName)) defer results.RecordResult(identifiers.TestShudtownIdentifier) shutdownTest(podNamespace, podName) @@ -304,10 +307,10 @@ func testPodsRecreation(env *config.TestEnvironment) { var nodesSorted []node // A slice version of nodes sorted by number of deployments descending ginkgo.It("Testing node draining effect of deployment", func() { env.SetNeedsRefresh() - for _, cut := range env.ContainersUnderTest { - namespace := cut.Oc.GetPodNamespace() - ginkgo.By(fmt.Sprintf("test deployment in namespace %s", namespace)) - deployments, notReadyDeployments = getDeployments(namespace) + for _, podUnderTest := range env.PodsUnderTest { + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("test deployment in namespace %s", podNamespace)) + deployments, notReadyDeployments = getDeployments(podNamespace) if len(deployments) == 0 { return } @@ -317,7 +320,7 @@ func testPodsRecreation(env *config.TestEnvironment) { } gomega.Expect(notReadyDeployments).To(gomega.BeEmpty()) ginkgo.By("Should return map of nodes to deployments") - nodesSorted = getDeploymentsNodes(namespace) + nodesSorted = getDeploymentsNodes(podNamespace) ginkgo.By("should create new replicas when node is drained") defer results.RecordResult(identifiers.TestPodRecreationIdentifier) testedDeployments := map[string]bool{} @@ -334,7 +337,7 @@ func testPodsRecreation(env *config.TestEnvironment) { // drain node drainNode(n.name) // should go in this // verify deployments are ready again - _, notReadyDeployments = getDeployments(namespace) + _, notReadyDeployments = getDeployments(podNamespace) gomega.Expect(notReadyDeployments).To(gomega.BeEmpty()) // this is to make sure pods are created again uncordonNode(n.name) if len(testedDeployments) == len(deployments) { @@ -423,8 +426,8 @@ func testPodAntiAffinity(env *config.TestEnvironment) { var deployments dp.DeploymentMap ginkgo.When("CNF is designed in high availability mode ", func() { ginkgo.It("Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment", func() { - for _, cut := range env.ContainersUnderTest { - podNamespace := cut.Oc.GetPodNamespace() + for _, podUnderTest := range env.PodsUnderTest { + podNamespace := podUnderTest.Namespace defer results.RecordResult(identifiers.TestPodHighAvailabilityBestPractices) deployments, _ = getDeployments(podNamespace) if len(deployments) == 0 { @@ -480,12 +483,12 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { func testOwner(env *config.TestEnvironment) { ginkgo.When("Testing owners of CNF pod", func() { ginkgo.It("Should be only ReplicaSet", func() { - for _, cut := range env.ContainersUnderTest { - podNamespace := cut.Oc.GetPodNamespace() - podName := cut.Oc.GetPodName() + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Should be ReplicaSet %s %s", podNamespace, podName)) defer results.RecordResult(identifiers.TestPodDeploymentBestPracticesIdentifier) - context := common.GetContext() tester := owners.NewOwners(common.DefaultTimeout, podNamespace, podName) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 8c2c5252a..9a0deed53 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -121,10 +121,10 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count func testNodePort(env *config.TestEnvironment) { ginkgo.It("Should not have services of type NodePort", func() { - for _, cut := range env.ContainersUnderTest { + for _, podUnderTest := range env.PodsUnderTest { defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) context := common.GetContext() - podNamespace := cut.Oc.GetPodNamespace() + podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing services in namespace %s", podNamespace)) tester := nodeport.NewNodePort(common.DefaultTimeout, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 411ff7e5a..1f1af32e7 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -148,9 +148,9 @@ func getPodNodeName(context *interactive.Context, podName, podNamespace string) return podNameTester.GetNodeName() } -func getCurrentKernelCmdlineArgs(targetPodOc *interactive.Oc) map[string]string { +func getCurrentKernelCmdlineArgs(targetContainerOc *interactive.Oc) map[string]string { currentKernelCmdlineArgsTester := currentkernelcmdlineargs.NewCurrentKernelCmdlineArgs(common.DefaultTimeout) - test, err := tnf.NewTest(targetPodOc.GetExpecter(), currentKernelCmdlineArgsTester, []reel.Handler{currentKernelCmdlineArgsTester}, targetPodOc.GetErrorChannel()) + test, err := tnf.NewTest(targetContainerOc.GetExpecter(), currentKernelCmdlineArgsTester, []reel.Handler{currentKernelCmdlineArgsTester}, targetContainerOc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) common.RunAndValidateTest(test) currnetKernelCmdlineArgs := currentKernelCmdlineArgsTester.GetKernelArguments() @@ -217,18 +217,18 @@ func testBootParams(env *config.TestEnvironment) { for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() podNameSpace := cut.Oc.GetPodNamespace() - targetPodOc := cut.Oc - testBootParamsHelper(context, podName, podNameSpace, targetPodOc) + targetContainerOc := cut.Oc + testBootParamsHelper(context, podName, podNameSpace, targetContainerOc) } }) } -func testBootParamsHelper(context *interactive.Context, podName, podNamespace string, targetPodOc *interactive.Oc) { +func testBootParamsHelper(context *interactive.Context, podName, podNamespace string, targetContainerOc *interactive.Oc) { ginkgo.By(fmt.Sprintf("Testing boot params for the pod's node %s/%s", podNamespace, podName)) defer results.RecordResult(identifiers.TestUnalteredStartupBootParamsIdentifier) nodeName := getPodNodeName(context, podName, podNamespace) mcName := getMcName(context, nodeName) mcKernelArgumentsMap := getMcKernelArguments(context, mcName) - currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetPodOc) + currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetContainerOc) grubKernelConfigMap := getGrubKernelArgs(context, nodeName) for key, mcVal := range mcKernelArgumentsMap { @@ -243,16 +243,16 @@ func testBootParamsHelper(context *interactive.Context, podName, podNamespace st func testSysctlConfigs(env *config.TestEnvironment) { ginkgo.It("platform-sysctl-config", func() { - context := common.GetContext() - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - podNameSpace := cut.Oc.GetPodNamespace() - testSysctlConfigsHelper(context, podName, podNameSpace) + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNameSpace := podUnderTest.Namespace + testSysctlConfigsHelper(podName, podNameSpace) } }) } -func testSysctlConfigsHelper(context *interactive.Context, podName, podNamespace string) { +func testSysctlConfigsHelper(podName, podNamespace string) { ginkgo.By(fmt.Sprintf("Testing sysctl config files for the pod's node %s/%s", podNamespace, podName)) + context := common.GetContext() nodeName := getPodNodeName(context, podName, podNamespace) combinedSysctlSettings := getSysctlConfigArgs(context, nodeName) mcName := getMcName(context, nodeName) From 0082808d60b1d6964ab30a8c6297a2abc7909d1f Mon Sep 17 00:00:00 2001 From: Isaak Dorfman Date: Thu, 19 Aug 2021 19:04:21 +0300 Subject: [PATCH 013/344] CNFCERT-30 changes (#304) --- .github/workflows/pre-main.yaml | 6 ++++-- Dockerfile | 19 ++++++++++++++++++- README.md | 9 +++++++++ run-cnf-suites.sh | 6 ++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 831535a75..4bd0efd59 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -20,6 +20,8 @@ env: TNF_SRC_URL: 'https://github.com/${{ github.repository }}' TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${IMAGE_NAME}:${IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}' NAMESPACE_TO_GENERATE: tnf + TNF_PARTNER_DIR: '/usr/tnf-partner' + TNF_PARTNER_SRC_DIR: '${TNF_PARTNER_DIR}/src' jobs: lint: @@ -43,7 +45,7 @@ jobs: - name: Install golint run: go get golang.org/x/lint/golint - # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However, + # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However # there does not appear to be a way to honor our existing .golangci.yml. For now, mimic developer behavior. - name: Install golangci-lint run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.39.0 @@ -145,7 +147,7 @@ jobs: run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} diagnostic - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} generic access-control + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} access-control lifecycle platform observablility networking affiliated-certification # Push the new unstable TNF image to Quay.io. diff --git a/Dockerfile b/Dockerfile index 9674d1c52..776a5ccfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ FROM registry.access.redhat.com/ubi8/ubi:latest AS build +ARG TNF_PARTNER_DIR=/usr/tnf-partner + +ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src ENV GOLANGCI_VERSION=v1.32.2 ENV OPENSHIFT_VERSION=4.6.32 @@ -45,11 +48,22 @@ ARG TNF_VERSION ARG TNF_SRC_URL=https://github.com/test-network-function/test-network-function ARG GIT_CHECKOUT_TARGET=$TNF_VERSION +# Git identifier to checkout for partner +ARG TNF_PARTNER_VERSION +ARG TNF_PARTNER_SRC_URL=https://github.com/test-network-function/cnf-certification-test-partner +ARG GIT_PARTNER_CHECKOUT_TARGET=$TNF_PARTNER_VERSION + + # Clone the TNF source repository and checkout the target branch/tag/commit RUN git clone --no-single-branch --depth=1 ${TNF_SRC_URL} ${TNF_SRC_DIR} RUN git -C ${TNF_SRC_DIR} fetch origin ${GIT_CHECKOUT_TARGET} RUN git -C ${TNF_SRC_DIR} checkout ${GIT_CHECKOUT_TARGET} +# Clone the partner source repository and checkout the target branch/tag/commit +RUN git clone --no-single-branch --depth=1 ${TNF_PARTNER_SRC_URL} ${TNF_PARTNER_SRC_DIR} +RUN git -C ${TNF_PARTNER_SRC_DIR} fetch origin ${GIT_PARTNER_CHECKOUT_TARGET} +RUN git -C ${TNF_PARTNER_SRC_DIR} checkout ${GIT_PARTNER_CHECKOUT_TARGET} + # Build TNF binary WORKDIR ${TNF_SRC_DIR} # TODO: RUN make install-tools @@ -72,7 +86,7 @@ WORKDIR ${TNF_DIR} RUN ln -s ${TNF_DIR}/config/testconfigure.yml ${TNF_DIR}/test-network-function/testconfigure.yml # Remove most of the build artefacts -RUN yum remove -y gcc git make wget && \ +RUN yum remove -y gcc git wget && \ yum clean all && \ rm -rf ${TNF_SRC_DIR} && \ rm -rf ${TEMP_DIR} && \ @@ -85,9 +99,12 @@ RUN yum remove -y gcc git make wget && \ # Copy the state into a new flattened image to reduce size. # TODO run as non-root FROM scratch +ARG TNF_PARTNER_DIR=/usr/tnf-partner COPY --from=build / / ENV TNF_CONFIGURATION_PATH=/usr/tnf/config/tnf_config.yml ENV KUBECONFIG=/usr/tnf/kubeconfig/config +ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src +ENV PATH="/usr/local/oc/bin:${PATH}" WORKDIR /usr/tnf ENV SHELL=/bin/bash CMD ["./run-cnf-suites.sh", "-o", "claim", "diagnostic", "generic"] diff --git a/README.md b/README.md index 32b4f99cc..1c62daa1d 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,15 @@ Likewise, to enable intrusive tests, set the following: export TNF_NON_INTRUSIVE_ONLY=false ``` +### Specifiy the location of the partner repo +This env var is mandatory. +You must clone the partner [repo](https://github.com/test-network-function/cnf-certification-test-partner) +and set TNF_PARTNER_SRC_DIR to point to it. + +```shell script +export TNF_PARTNER_SRC_DIR=~/code/cnf-certification-test-partner +``` + ### Execute test suites from openshift-kni/cnf-feature-deploy The test suites from openshift-kni/cnf-feature-deploy can be run prior to the actual CNF certification test execution and the results are incorporated in the same claim file if the following environment variable is set: diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index ac9a85764..f731be3e6 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -66,5 +66,11 @@ if [[ ! -f "/proc/1/cgroup" ]] || grep -q init\.scope /proc/1/cgroup; then cd .. fi +if [[ -z "${TNF_PARTNER_SRC_DIR}" ]]; then + echo "env var \"TNF_PARTNER_SRC_DIR\" not set, running the script without updating infra" +else + make -C $TNF_PARTNER_SRC_DIR install-partner-pods +fi + echo "Running with focus '$FOCUS'. Report will be output to '$OUTPUT_LOC'" cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" ${GINKGO_ARGS} From 5b5f2ea881773d235b8e0890505ec977ff245020 Mon Sep 17 00:00:00 2001 From: wying3 <81822258+wying3@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:40:27 -0500 Subject: [PATCH 014/344] List all installed CSI drivers info from cluster CNFCERT24-item 1 (#309) --- go.mod | 3 +- go.sum | 7 + pkg/tnf/handlers/csidriver/csidriver.json | 24 ++++ pkg/tnf/handlers/csidriver/csidriver_test.go | 122 ++++++++++++++++++ pkg/tnf/handlers/csidriver/doc.go | 18 +++ pkg/tnf/identifier/identifiers.go | 22 +++- test-network-function/diagnostic/suite.go | 46 +++++++ .../identifiers/identifiers.go | 11 ++ test-network-function/suite_test.go | 2 + 9 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 pkg/tnf/handlers/csidriver/csidriver.json create mode 100644 pkg/tnf/handlers/csidriver/csidriver_test.go create mode 100644 pkg/tnf/handlers/csidriver/doc.go diff --git a/go.mod b/go.mod index 2c09bb922..31647491d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,8 @@ require ( github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.39.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 055a310f4..ab9ec6ae0 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -253,6 +254,7 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -322,6 +324,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -333,6 +336,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -511,6 +515,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -608,6 +613,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/tnf/handlers/csidriver/csidriver.json b/pkg/tnf/handlers/csidriver/csidriver.json new file mode 100644 index 000000000..50237b888 --- /dev/null +++ b/pkg/tnf/handlers/csidriver/csidriver.json @@ -0,0 +1,24 @@ +{ + "identifier": { + "url": "http://test-network-function.com/tests/csiDriver", + "version": "v1.0.0" + }, + "description": "This test checks third party CSI driver installed in cluster for cnf.", + "testResult": 0, + "testTimeout": 10000000000, + "reelFirstStep": { + "execute": + "oc get csidriver -o json\n", + "expect": [ + "(?m)(.|\n)+" + ], + "timeout": 10000000000 + }, + "resultContexts": [ + { + "pattern": "(?m)(.|\n)+", + "defaultResult": 1 + } + ] + } + \ No newline at end of file diff --git a/pkg/tnf/handlers/csidriver/csidriver_test.go b/pkg/tnf/handlers/csidriver/csidriver_test.go new file mode 100644 index 000000000..42afe6149 --- /dev/null +++ b/pkg/tnf/handlers/csidriver/csidriver_test.go @@ -0,0 +1,122 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package csidriver_test + +import ( + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/xeipuuv/gojsonschema" +) + +const ( + testTimeoutDuration = time.Second * 10 +) + +var ( + genericTestSchemaFile = path.Join("schemas", "generic-test.schema.json") + csiDriverFilename = "csidriver.json" + /* #nosec G101 */ + expectedPassPattern = "(?m)(.|\n)+" + pathRelativeToRoot = path.Join("..", "..", "..", "..") + pathToTestSchemaFile = path.Join(pathRelativeToRoot, genericTestSchemaFile) +) + +func createTest() (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { + return generic.NewGenericFromJSONFile(csiDriverFilename, pathToTestSchemaFile) +} + +func TestCSIs_Args(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Nil(t, (*test).Args()) +} + +func TestCSIs_GetIdentifier(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, identifier.CSIDriverIdentifier, (*test).GetIdentifier()) +} + +func TestCSIs_ReelFirst(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + step := handler.ReelFirst() + assert.Equal(t, "oc get csidriver -o json\n", step.Execute) + assert.Equal(t, []string{expectedPassPattern}, step.Expect) + assert.Equal(t, testTimeoutDuration, step.Timeout) +} + +func TestCSIs_ReelEOF(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + // just ensure there isn't a panic + handler.ReelEOF() +} + +func TestCSIs_ReelTimeout(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + assert.Nil(t, handler.ReelTimeout()) +} + +func TestCSIs_ReelMatch(t *testing.T) { + tester, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + + // Positive Test + step := handler.ReelMatch(expectedPassPattern, "", "anythingMatches") + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, (*tester).Result()) +} diff --git a/pkg/tnf/handlers/csidriver/doc.go b/pkg/tnf/handlers/csidriver/doc.go new file mode 100644 index 000000000..b730d3ab6 --- /dev/null +++ b/pkg/tnf/handlers/csidriver/doc.go @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package csidriver provides a test for reading the CNF cluster csi driver info +package csidriver diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 3f74c0a8e..9909f6a13 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -57,8 +57,8 @@ const ( podantiaffinityIdentifierURL = "http://test-network-function.com/tests/testPodHighAvailability" shutdownIdentifierURL = "http://test-network-function.com/tests/shutdown" scalingIdentifierURL = "http://test-network-function.com/tests/scaling" - - versionOne = "v1.0.0" + csiDriverIdentifierURL = "http://test-network-function.com/tests/csiDriver" + versionOne = "v1.0.0" ) const ( @@ -584,6 +584,18 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.OcBinaryName, }, }, + csiDriverIdentifierURL: { + Identifier: CSIDriverIdentifier, + Description: "extracts the csi driver info in the cluser", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + }, + }, } // HostnameIdentifier is the Identifier used to represent the generic hostname test case. @@ -814,3 +826,9 @@ var ScalingIdentifier = Identifier{ URL: scalingIdentifierURL, SemanticVersion: versionOne, } + +// CSIDriverIdentifier is the Identifier used to represent the CSI driver test case. +var CSIDriverIdentifier = Identifier{ + URL: csiDriverIdentifierURL, + SemanticVersion: versionOne, +} diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index e17caed75..361ee89dd 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -35,9 +35,18 @@ var ( nodesHwInfo = NodesHwInfo{} + // csiDriver stores the csi driver JSON output of `oc get csidriver -o json` + csiDriver = make(map[string]interface{}) + // nodesTestPath is the file location of the nodes.json test case relative to the project root. nodesTestPath = path.Join("pkg", "tnf", "handlers", "node", "nodes.json") + // csiDriverTestPath is the file location of the csidriver.json test case relative to the project root. + csiDriverTestPath = path.Join("pkg", "tnf", "handlers", "csidriver", "csidriver.json") + + // relativeCsiDriverTestPath is the relative path to the csidriver.json test case. + relativeCsiDriverTestPath = path.Join(pathRelativeToRoot, csiDriverTestPath) + // pathRelativeToRoot is used to calculate relative filepaths for the `test-network-function` executable entrypoint. pathRelativeToRoot = path.Join("..") @@ -88,6 +97,10 @@ var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { defer results.RecordResult(identifiers.TestNodesHwInfoIdentifier) testNodesHwInfo() }) + ginkgo.It("should report cluster CSI driver info", func() { + defer results.RecordResult(identifiers.TestClusterCsiInfoIdentifier) + listClusterCSIInfo() + }) }) }) @@ -127,6 +140,11 @@ func GetNodesHwInfo() NodesHwInfo { return nodesHwInfo } +// GetCsiDriverInfo returns the CSI driver info of running `oc get csidriver -o json`. +func GetCsiDriverInfo() map[string]interface{} { + return csiDriver +} + func getFirstNode(labelFilter map[string]*string) string { context := common.GetContext() tester := nodenames.NewNodeNames(defaultTestTimeout, labelFilter) @@ -271,3 +289,31 @@ func getNodeLspci(nodeName string) []string { gomega.Expect(err).To(gomega.BeNil()) return tester.Processed } + +// check CSI driver info in cluster +func listClusterCSIInfo() { + if common.IsMinikube() { + ginkgo.Skip("CSI is not checked in minikube") + } + context := common.GetContext() + test, handlers, result, err := generic.NewGenericFromJSONFile(relativeCsiDriverTestPath, common.RelativeSchemaPath) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(len(handlers)).To(gomega.Equal(1)) + gomega.Expect(test).ToNot(gomega.BeNil()) + tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + testResult, err := tester.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + genericTest := (*test).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + match := genericTest.GetMatches()[0] + err = json.Unmarshal([]byte(match.Match), &csiDriver) + gomega.Expect(err).To(gomega.BeNil()) +} diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index b443e3398..014985fd7 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -189,6 +189,11 @@ var ( Url: formTestURL(common.LifecycleTestKey, "scaling"), Version: versionOne, } + // TestClusterCsiInfoIdentifier list Cluster CSIdriver Identifier retrieves Third Party CSI driver info. + TestClusterCsiInfoIdentifier = claim.Identifier{ + Url: formTestURL(common.DiagnosticTestKey, "cluster-csi-info"), + Version: versionOne, + } ) func formDescription(identifier claim.Identifier, description string) string { @@ -489,4 +494,10 @@ the changes for you.`, scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s.`), Remediation: `Make sure CNF deployments/replica sets can scale in/out successfully.`, }, + TestClusterCsiInfoIdentifier: { + Identifier: TestClusterCsiInfoIdentifier, + Type: informativeResult, + Description: formDescription(TestClusterCsiInfoIdentifier, + `extracts CSI driver information in the cluster.`), + }, } diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 0aea6e1bf..02294d368 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -223,10 +223,12 @@ func generateNodes() map[string]interface{} { nodeSummaryField = "nodeSummary" cniPluginsField = "cniPlugins" nodesHwInfo = "nodesHwInfo" + csiDriverInfo = "csiDriver" ) nodes := map[string]interface{}{} nodes[nodeSummaryField] = diagnostic.GetNodeSummary() nodes[cniPluginsField] = diagnostic.GetCniPlugins() nodes[nodesHwInfo] = diagnostic.GetNodesHwInfo() + nodes[csiDriverInfo] = diagnostic.GetCsiDriverInfo() return nodes } From 7cc48f42114d2b47e36670641969cc3ed1544e77 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Mon, 23 Aug 2021 11:40:11 -0400 Subject: [PATCH 015/344] Shamadi node draning fix (#314) --- CATALOG.md | 18 ++++++++ .../deploymentsdrain/deploymentsdrain.go | 14 ++++-- run-cnf-suites.sh | 2 +- test-network-function/accesscontrol/suite.go | 2 +- test-network-function/lifecycle/suite.go | 45 ++++++++++--------- 5 files changed, 54 insertions(+), 27 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 3e33ec62b..4075f2f62 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -62,6 +62,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/operator-is-certified tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP). Result Type|normative Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP). +### http://test-network-function.com/testcases/diagnostic/cluster-csi-info + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/diagnostic/cluster-csi-info extracts CSI driver information in the cluster. +Result Type|informative +Suggested Remediation| ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description @@ -239,6 +247,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`jq`, `oc` +### http://test-network-function.com/tests/csiDriver +Property|Description +---|--- +Version|v1.0.0 +Description|extracts the csi driver info in the cluser +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`oc` + ### http://test-network-function.com/tests/currentKernelCmdlineArgs Property|Description ---|--- diff --git a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go index 1eafd5814..cb9a826fc 100644 --- a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go +++ b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go @@ -17,6 +17,7 @@ package deploymentsdrain import ( + "strings" "time" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -34,20 +35,22 @@ type DeploymentsDrain struct { result int timeout time.Duration args []string + node string } // NewDeploymentsDrain creates a new DeploymentsDrain tnf.Test. -func NewDeploymentsDrain(timeout time.Duration, node string) *DeploymentsDrain { +func NewDeploymentsDrain(timeout time.Duration, nodeName string) *DeploymentsDrain { drainTimeout := timeout * drainTimeoutPercentage / 100 drainTimeoutString := drainTimeout.String() return &DeploymentsDrain{ timeout: timeout, result: tnf.ERROR, args: []string{ - "oc", "adm", "drain", node, "--pod-selector=pod-template-hash", "--disable-eviction=true", + "oc", "adm", "drain", nodeName, "--pod-selector=pod-template-hash", "--disable-eviction=true", "--delete-local-data=true", "--ignore-daemonsets=true", "--timeout=" + drainTimeoutString, "&&", "echo", "SUCCESS", }, + node: nodeName, } } @@ -87,7 +90,12 @@ func (dd *DeploymentsDrain) ReelMatch(_, _, _ string) *reel.Step { // ReelTimeout does nothing; no action is necessary upon timeout. func (dd *DeploymentsDrain) ReelTimeout() *reel.Step { - return nil + str := []string{"oc", "adm", "uncordon", dd.node} + return &reel.Step{ + Expect: []string{"(?m).*uncordoned"}, + Timeout: dd.timeout, + Execute: strings.Join(str, " "), + } } // ReelEOF does nothing; no action is necessary upon EOF. diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index f731be3e6..bb36249a5 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -73,4 +73,4 @@ else fi echo "Running with focus '$FOCUS'. Report will be output to '$OUTPUT_LOC'" -cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" ${GINKGO_ARGS} +cd ./test-network-function && ./test-network-function.test -ginkgo.v -ginkgo.focus="$FOCUS" ${GINKGO_ARGS} diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 208e905fa..e139d097e 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -52,7 +52,7 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { // Run the tests that interact with the pods ginkgo.When("under test", func() { - var allTests []string = testcases.GetConfiguredPodTests() + allTests := testcases.GetConfiguredPodTests() for _, testType := range allTests { testFile, err := testcases.LoadConfiguredTestFile(common.ConfiguredTestFile) gomega.Expect(testFile).ToNot(gomega.BeNil()) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 2318ea11e..5e7d255dc 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -99,7 +99,6 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { testShutdown(env) testPodAntiAffinity(env) - if !common.NonIntrusive() { testPodsRecreation(env) @@ -304,7 +303,8 @@ func shutdownTest(podNamespace, podName string) { func testPodsRecreation(env *config.TestEnvironment) { var deployments dp.DeploymentMap var notReadyDeployments []string - var nodesSorted []node // A slice version of nodes sorted by number of deployments descending + nodesNames := make(map[string]node) + namespaces := make(map[string]bool) ginkgo.It("Testing node draining effect of deployment", func() { env.SetNeedsRefresh() for _, podUnderTest := range env.PodsUnderTest { @@ -314,36 +314,37 @@ func testPodsRecreation(env *config.TestEnvironment) { if len(deployments) == 0 { return } + if _, exists := namespaces[podNamespace]; exists { + continue + } + namespaces[podNamespace] = true // We require that all deployments have the desired number of replicas and are all up to date if len(notReadyDeployments) != 0 { ginkgo.Skip("Can not test when deployments are not ready") } gomega.Expect(notReadyDeployments).To(gomega.BeEmpty()) ginkgo.By("Should return map of nodes to deployments") - nodesSorted = getDeploymentsNodes(podNamespace) - ginkgo.By("should create new replicas when node is drained") - defer results.RecordResult(identifiers.TestPodRecreationIdentifier) - testedDeployments := map[string]bool{} + nodesSorted := getDeploymentsNodes(podNamespace) for _, n := range nodesSorted { - oldLen := len(testedDeployments) // this starts with zero - // mark tested deployments - for d := range n.deployments { - testedDeployments[d] = true + if _, exists := nodesNames[n.name]; !exists { + nodesNames[n.name] = n } - if oldLen == len(testedDeployments) { - // If node does not add new deployments then skip it - continue - } - // drain node - drainNode(n.name) // should go in this - // verify deployments are ready again - _, notReadyDeployments = getDeployments(podNamespace) - gomega.Expect(notReadyDeployments).To(gomega.BeEmpty()) // this is to make sure pods are created again - uncordonNode(n.name) - if len(testedDeployments) == len(deployments) { - break + } + } + ginkgo.By("should create new replicas when node is drained") + defer results.RecordResult(identifiers.TestPodRecreationIdentifier) + for _, n := range nodesNames { + // drain node + drainNode(n.name) // should go in this + // verify deployments are ready again + for namespace := range namespaces { + _, notReadyDeployments = getDeployments(namespace) + if len(notReadyDeployments) != 0 { + uncordonNode(n.name) + ginkgo.Fail(fmt.Sprintf("did not create replicas when noede %s is drained", n.name)) } } + uncordonNode(n.name) } }) } From 884445bf6a0bb24ec64252b705a88d62abcf51b9 Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:41:50 +0300 Subject: [PATCH 016/344] Add_test_identifier_for_isRedhatRelease (#308) --- test-network-function/generic/suite.go | 37 ------------------- .../identifiers/identifiers.go | 12 ++++++ test-network-function/platform/suite.go | 27 ++++++++++++++ 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/test-network-function/generic/suite.go b/test-network-function/generic/suite.go index 1d3a1985d..a12363db2 100644 --- a/test-network-function/generic/suite.go +++ b/test-network-function/generic/suite.go @@ -17,29 +17,17 @@ package generic import ( - "fmt" - "github.com/test-network-function/test-network-function/pkg/config" - "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" - "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" - "github.com/test-network-function/test-network-function/test-network-function/common" - "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" - "github.com/onsi/gomega" ) const ( testsKey = "generic" ) -// -// All actual test code belongs below here. Utilities belong above. -// - // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(testsKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testsKey) { @@ -47,30 +35,5 @@ var _ = ginkgo.Describe(testsKey, func() { ginkgo.BeforeEach(func() { env.LoadAndRefresh() }) - - testIsRedHatRelease(env) } }) - -// testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. -func testIsRedHatRelease(env *config.TestEnvironment) { - ginkgo.It("Should report a proper Red Hat version", func() { - for _, cut := range env.ContainersUnderTest { - testContainerIsRedHatRelease(cut) - } - }) -} - -// testContainerIsRedHatRelease tests whether the container attached to oc is Red Hat based. -func testContainerIsRedHatRelease(cut *config.Container) { - podName := cut.Oc.GetPodName() - containerName := cut.Oc.GetPodContainerName() - context := cut.Oc - ginkgo.By(fmt.Sprintf("%s(%s) is checked for Red Hat version", podName, containerName)) - versionTester := redhat.NewRelease(common.DefaultTimeout) - test, err := tnf.NewTest(context.GetExpecter(), versionTester, []reel.Handler{versionTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) -} diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 014985fd7..86d720c22 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -189,6 +189,11 @@ var ( Url: formTestURL(common.LifecycleTestKey, "scaling"), Version: versionOne, } + // TestIsRedHatReleaseIdentifier ensures platform is defined + TestIsRedHatReleaseIdentifier = claim.Identifier{ + Url: formTestURL(common.PlatformAlterationTestKey, "isredhat-release"), + Version: versionOne, + } // TestClusterCsiInfoIdentifier list Cluster CSIdriver Identifier retrieves Third Party CSI driver info. TestClusterCsiInfoIdentifier = claim.Identifier{ Url: formTestURL(common.DiagnosticTestKey, "cluster-csi-info"), @@ -494,6 +499,13 @@ the changes for you.`, scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s.`), Remediation: `Make sure CNF deployments/replica sets can scale in/out successfully.`, }, + TestIsRedHatReleaseIdentifier: { + Identifier: TestIsRedHatReleaseIdentifier, + Type: normativeResult, + Description: formDescription(TestIsRedHatReleaseIdentifier, + `The test verifies if the container base image is redhat.`), + Remediation: `build a new docker image that's based on UBI (redhat universal base image).`, + }, TestClusterCsiInfoIdentifier: { Identifier: TestClusterCsiInfoIdentifier, Type: informativeResult, diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 1f1af32e7..4d11c41e2 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -34,6 +34,7 @@ import ( "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/cnffsdiff" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/containerid" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/currentkernelcmdlineargs" @@ -78,9 +79,35 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { testSysctlConfigs(env) } + testIsRedHatRelease(env) + } }) +// testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. +func testIsRedHatRelease(env *config.TestEnvironment) { + ginkgo.It("Should report a proper Red Hat version", func() { + defer results.RecordResult(identifiers.TestIsRedHatReleaseIdentifier) + for _, cut := range env.ContainersUnderTest { + testContainerIsRedHatRelease(cut) + } + }) +} + +// testContainerIsRedHatRelease tests whether the container attached to oc is Red Hat based. +func testContainerIsRedHatRelease(cut *config.Container) { + podName := cut.Oc.GetPodName() + containerName := cut.Oc.GetPodContainerName() + context := cut.Oc + ginkgo.By(fmt.Sprintf("%s(%s) is checked for Red Hat version", podName, containerName)) + versionTester := redhat.NewRelease(common.DefaultTimeout) + test, err := tnf.NewTest(context.GetExpecter(), versionTester, []reel.Handler{versionTester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) +} + // testContainersFsDiff test that all CUT didn't install new packages are starting func testContainersFsDiff(env *config.TestEnvironment) { ginkgo.It("platform-fsdiff", func() { From e8267261b86e461b2c4b710f2f574ea030363cef Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:51:02 -0500 Subject: [PATCH 017/344] Fix for hostPath test not working (#317) --- pkg/tnf/testcases/data/cnf/privilegedpod.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tnf/testcases/data/cnf/privilegedpod.go b/pkg/tnf/testcases/data/cnf/privilegedpod.go index e326d78c1..8925a60a2 100644 --- a/pkg/tnf/testcases/data/cnf/privilegedpod.go +++ b/pkg/tnf/testcases/data/cnf/privilegedpod.go @@ -43,10 +43,10 @@ var PrivilegedPodJSON = string(`{ "name": "HOST_PATH_CHECK", "skiptest": true, "loop": 0, - "command": "oc get pod %s -n %s -o json | jq -r '.spec.hostpath.path'", + "command": "oc get pods %s -n %s -o go-template='{{ range .spec.volumes}}{{.hostPath.path}}{{end}}'", "action": "allow", "expectedstatus": [ - "NULL_FALSE" + "^()*$" ] }, { From 5d0c4c7a960284b9862b61f98d422128ef05dad2 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Tue, 24 Aug 2021 09:05:03 -0400 Subject: [PATCH 018/344] Update tests naming (#312) Enable focus and skip specific tests Update Readme Add verbose mode --- .github/workflows/pre-main.yaml | 8 +- CATALOG.md | 8 ++ Dockerfile | 2 +- README.md | 16 ++- run-cnf-suites.sh | 59 ++++---- run-tnf-container.sh | 74 +++++++--- script/run-container.sh | 2 +- test-network-function/accesscontrol/suite.go | 20 ++- test-network-function/certification/suite.go | 43 +++--- test-network-function/common/env.go | 6 +- test-network-function/diagnostic/suite.go | 91 ++++++------ .../identifiers/identifiers.go | 7 + test-network-function/lifecycle/suite.go | 123 +++++++++-------- test-network-function/networking/suite.go | 17 ++- test-network-function/observability/suite.go | 15 +- test-network-function/operator/suite.go | 31 +++-- test-network-function/platform/suite.go | 129 +++++++++--------- 17 files changed, 368 insertions(+), 283 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 4bd0efd59..52b4df3e1 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -120,10 +120,10 @@ jobs: # Perform smoke tests. - name: 'Test: Run diagnostic test suite' - run: ./run-cnf-suites.sh diagnostic + run: ./run-cnf-suites.sh --focus diagnostic - name: 'Test: Run test suites' - run: ./run-cnf-suites.sh access-control lifecycle platform observablility networking affiliated-certification + run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification # Perform smoke tests using a TNF container. @@ -144,10 +144,10 @@ jobs: shell: bash - name: 'Test: Run diagnostic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} diagnostic + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} access-control lifecycle platform observablility networking affiliated-certification + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification # Push the new unstable TNF image to Quay.io. diff --git a/CATALOG.md b/CATALOG.md index 4075f2f62..c22f52c4b 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -206,6 +206,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. +### http://test-network-function.com/testcases/platform-alteration/isredhat-release + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release The test verifies if the container base image is redhat. +Result Type|normative +Suggested Remediation|build a new docker image that's based on UBI (redhat universal base image). ### http://test-network-function.com/testcases/platform-alteration/sysctl-config Property|Description diff --git a/Dockerfile b/Dockerfile index 776a5ccfa..ef980a4b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,4 +107,4 @@ ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src ENV PATH="/usr/local/oc/bin:${PATH}" WORKDIR /usr/tnf ENV SHELL=/bin/bash -CMD ["./run-cnf-suites.sh", "-o", "claim", "diagnostic", "generic"] +CMD ["./run-cnf-suites.sh", "-o", "claim", "-f", "diagnostic"] diff --git a/README.md b/README.md index 1c62daa1d..3f0ff22e8 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ To pull the latest container and run the tests you use the following command. Th * `-t` gives the local directory that contains tnf config files set up for the test. * `-o` gives the local directory that the test results will be available in once the container exits. +* `-f` gives the list of suites that should be executed * Finally, list the specs to be run must be specified, space-separated. Optional arguments are: @@ -153,12 +154,13 @@ Optional arguments are: * `-i` gives a name to a custom TNF container image. Supports local images, as well as images from external registries. * `-k` gives a path to one or more kubeconfig files soto be used by the container to authenticate with the cluster. Paths must be separated by a colon. * `-n` gives the network mode of the container. Defaults to `bridge`. See the [docker run --network parameter reference](https://docs.docker.com/engine/reference/run/#network-settings) for more information on how to configure network settings. +* `-s` gives the name of tests that should be skipped If `-k` is not specified, autodiscovery is performed. The autodiscovery first looks for paths in the `$KUBECONFIG` environment variable on the host system, and if the variable is not set or is empty, the default configuration stored in `$HOME/.kube/config` is checked. ```shell script -./run-tnf-container.sh -k ~/.kube/config -t ~/tnf/config -o ~/tnf/output diagnostic access-control +./run-tnf-container.sh -k ~/.kube/config -t ~/tnf/config -o ~/tnf/output -f diagnostic access-control -s access-control-host-resource-PRIVILEGED_POD ``` *Note*: Tests must be specified after all other arguments! see [General tests](#general-tests) for a list of available keywords. @@ -197,7 +199,7 @@ docker build -t test-network-function:v1.0.5 \ To make `run-tnf-container.sh` use the newly built image, specify the custom TNF image using the `-i` parameter. ```shell script -./run-tnf-container.sh -i test-network-function:v1.0.5 -t ~/tnf/config -o ~/tnf/output diagnostic access-control +./run-tnf-container.sh -i test-network-function:v1.0.5 -t ~/tnf/config -o ~/tnf/output -f diagnostic access-control ``` Note: see [General tests](#general-tests) for a list of available keywords. @@ -272,11 +274,11 @@ script. Run any combination of the suites keywords listed at in the [General tests](#general-tests) section, e.g. ```shell script -./run-cnf-suites.sh diagnostic -./run-cnf-suites.sh diagnostic lifecycle -./run-cnf-suites.sh diagnostic networking operator -./run-cnf-suites.sh diagnostic platform-alteration -./run-cnf-suites.sh diagnostic generic lifecycle affiliated-certification operator +./run-cnf-suites.sh -f diagnostic +./run-cnf-suites.sh -f diagnostic lifecycle +./run-cnf-suites.sh -f diagnostic networking operator +./run-cnf-suites.sh -f diagnostic platform-alteration +./run-cnf-suites.sh -f diagnostic generic lifecycle affiliated-certification operator ``` By default the claim file will be output into the same location as the test executable. The `-o` argument for diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index bb36249a5..ba8a97837 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -4,10 +4,10 @@ export OUTPUT_LOC="$PWD/test-network-function" usage() { - echo "$0 [-o OUTPUT_LOC] SUITE [... SUITE]" + echo "$0 [-o OUTPUT_LOC] [-f SUITE...] -s [SUITE...]" echo "Call the script and list the test suites to run" echo " e.g." - echo " $0 [ARGS] access-control lifecycle" + echo " $0 [ARGS] -f access-control lifecycle" echo " will run the access-control and lifecycle suites" echo "" echo "Allowed suites are listed in the README." @@ -18,45 +18,41 @@ usage_error() { exit 1 } +FOCUS="" +SKIP="" # Parge args beginning with "-" while [[ $1 == -* ]]; do case "$1" in -h|--help|-\?) usage; exit 0;; -o) if (($# > 1)); then - OUTPUT_LOC=$2; shift 2 - else - echo "-o requires an argument" 1>&2 - exit 1 - fi ;; - --) shift; break;; - -*) echo "invalid option: $1" 1>&2; usage_error;; + OUTPUT_LOC=$2; shift + else + echo "-o requires an argument" 1>&2 + exit 1 + fi ;; + -s|--skip) + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + SKIP="$2|$SKIP" + shift + done;; + -f|--focus) + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + FOCUS="$2|$FOCUS" + shift + done;; + -*) echo "invalid option: $1" 1>&2; usage_error;; esac + shift done # specify Junit report file name. -GINKGO_ARGS="-ginkgo.v -junit $OUTPUT_LOC -claimloc $OUTPUT_LOC -ginkgo.reportFile $OUTPUT_LOC/cnf-certification-tests_junit.xml" -FOCUS="" +GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC -ginkgo.reportFile $OUTPUT_LOC/cnf-certification-tests_junit.xml -ginkgo.v -test.v" -for var in "$@" -do - FOCUS="$var|$FOCUS" - # case "$var" in - # diagnostic) FOCUS="diagnostic|$FOCUS";; - # access-control) FOCUS="ac|$FOCUS";; - # affiliated-certification) FOCUS="affiliated-certification|$FOCUS";; - # lifecycle) FOCUS="lifecycle|$FOCUS";; - # platform-alteration) FOCUS="platform-alteration|$FOCUS";; - # generic) FOCUS="generic|$FOCUS";; - # observability) FOCUS="observability|$FOCUS";; - # operator) FOCUS="operator|$FOCUS";; - # networking) FOCUS="networking|$FOCUS";; - # *) usage_error;; - # esac -done # If no focus is set then display usage and quit with a non-zero exit code. -[ -z "$FOCUS" ] && usage_error +[ -z "$FOCUS" ] && echo "no focus found" && usage_error FOCUS=${FOCUS%?} # strip the trailing "|" from the concatenation +SKIP=${SKIP%?} # strip the trailing "|" from the concatenation # Run cnf-feature-deploy test container if not running inside a container # cgroup file doesn't exist on MacOS. Consider that as not running in container as well @@ -72,5 +68,8 @@ else make -C $TNF_PARTNER_SRC_DIR install-partner-pods fi -echo "Running with focus '$FOCUS'. Report will be output to '$OUTPUT_LOC'" -cd ./test-network-function && ./test-network-function.test -ginkgo.v -ginkgo.focus="$FOCUS" ${GINKGO_ARGS} +echo "Running with focus '$FOCUS'" +echo "Running with skip '$SKIP'" +echo "Report will be output to '$OUTPUT_LOC'" + +cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" -ginkgo.skip="$SKIP" ${GINKGO_ARGS} diff --git a/run-tnf-container.sh b/run-tnf-container.sh index e0d3dd00c..3120bf5e5 100755 --- a/run-tnf-container.sh +++ b/run-tnf-container.sh @@ -27,10 +27,12 @@ export TNF_OFFICIAL_ORG=quay.io/testnetworkfunction/ export TNF_OFFICIAL_IMAGE="${TNF_OFFICIAL_ORG}${TNF_IMAGE_NAME}:${TNF_IMAGE_TAG}" export TNF_CMD="./run-cnf-suites.sh" export OUTPUT_ARG="-o" +export FOCUS_ARG="-f" +export SKIP_ARG="-s" usage() { read -d '' usage_prompt <<- EOF - Usage: $0 -t TNFCONFIG -o OUTPUT_LOC [-i IMAGE] [-k KUBECONFIG] [-n NETWORK_MODE] [-d DNS_RESOLVER_ADDRESS] SUITE [... SUITE] + Usage: $0 -t TNFCONFIG -o OUTPUT_LOC [-i IMAGE] [-k KUBECONFIG] [-n NETWORK_MODE] [-d DNS_RESOLVER_ADDRESS] -f SUITE [... SUITE] Configure and run the containerised TNF test offering. @@ -46,6 +48,8 @@ usage() { -n: set the network mode of the container. -d: set the DNS resolver address for the test containers started by docker, may be required with certain docker version if the kubeconfig contains host names + -f: Set the suites that should be tested, multiple suites can be supplied + -s: Set the test cases that should be skipped Kubeconfig lookup order 1. If -k is specified, use the paths provided with the -k option. @@ -54,21 +58,21 @@ usage() { (currently: $HOME/.kube/config). Examples - $0 -t ~/tnf/config -o ~/tnf/output diagnostic generic + $0 -t ~/tnf/config -o ~/tnf/output -f diagnostic networking -s diagnostic-nodes-hw-info Because -k is omitted, $(basename $0) will first try to autodiscover local kubeconfig files. - If it succeeds, the diagnostic and generic tests will be run using the autodiscovered configuration. + If it succeeds, the diagnostic and networking tests will be run using the autodiscovered configuration. The test results will be saved to the '~/tnf/output' directory on the host. - $0 -k ~/.kube/ABC:~/.kube/DEF -t ~/tnf/config -o ~/tnf/output diagnostic generic + $0 -k ~/.kube/ABC:~/.kube/DEF -t ~/tnf/config -o ~/tnf/output -f diagnostic networking The command will bind two kubeconfig files (~/.kube/ABC and ~/.kube/DEF) to the TNF container, - run the diagnostic and generic tests, and save the test results into the '~/tnf/output' directory + run the diagnostic and networking tests, and save the test results into the '~/tnf/output' directory on the host. - $0 -i custom-tnf-image:v1.2-dev -t ~/tnf/config -o ~/tnf/output diagnostic generic + $0 -i custom-tnf-image:v1.2-dev -t ~/tnf/config -o ~/tnf/output -f diagnostic networking - The command will run the diagnostic and generic tests as implemented in the custom-tnf-image:v1.2-dev + The command will run the diagnostic and networking tests as implemented in the custom-tnf-image:v1.2-dev local image set by the -i parameter. The test results will be saved to the '~/tnf/output' directory. Test suites @@ -132,50 +136,80 @@ perform_kubeconfig_autodiscovery # Parge args beginning with - while [[ $1 == -* ]]; do - echo "$1 $2" case "$1" in -h|--help|-\?) usage; exit 0;; -k) if (($# > 1)); then export LOCAL_KUBECONFIG=$2 unset kubeconfig_autodiscovery_source - shift 2 + shift else echo "-k requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-k $LOCAL_KUBECONFIG" + ;; -t) if (($# > 1)); then - export LOCAL_TNF_CONFIG=$2; shift 2 + export LOCAL_TNF_CONFIG=$2; shift else echo "-t requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-t $LOCAL_TNF_CONFIG" + ;; -o) if (($# > 1)); then - export OUTPUT_LOC=$2; shift 2 + export OUTPUT_LOC=$2; shift else echo "-o requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-o $OUTPUT_LOC" + ;; -i) if (($# > 1)); then - export TNF_IMAGE=$2; shift 2 + export TNF_IMAGE=$2; shift else echo "-i requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-i $TNF_IMAGE" + ;; -n) if (($# > 1)); then - export CONTAINER_NETWORK_MODE=$2; shift 2 + export CONTAINER_NETWORK_MODE=$2; shift else echo "-n requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-n CONTAINER_NETWORK_MODE" + ;; -d) if (($# > 1)); then export DNS_ARG=$2; shift 2 else echo "-d requires an argument" 1>&2 exit 1 - fi ;; + fi + echo "-d $DNS_ARGS" + ;; + -s) + TNF_SKIP_SUITES="" + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + TNF_SKIP_SUITES="$2 $TNF_SKIP_SUITES" + shift + done + export TNF_SKIP_SUITES + echo "-s $TNF_SKIP_SUITES" + ;; + -f) + TNF_FOCUS_SUITES="" + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + TNF_FOCUS_SUITES="$2 $TNF_FOCUS_SUITES" + shift + done + export TNF_FOCUS_SUITES + echo "-f $TNF_FOCUS_SUITES" + ;; --) shift; break;; -*) echo "invalid option: $1" 1>&2; usage_error;; esac + shift done display_kubeconfig_autodiscovery_summary @@ -185,4 +219,4 @@ cd script ./run-cfd-container.sh -./run-container.sh "$@" \ No newline at end of file +./run-container.sh "$@" \ No newline at end of file diff --git a/script/run-container.sh b/script/run-container.sh index 5e9c9f6b1..113b1dc78 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -108,4 +108,4 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ - $TNF_CMD $OUTPUT_ARG $CONTAINER_TNF_DIR/claim "$@" + $TNF_CMD $OUTPUT_ARG $CONTAINER_TNF_DIR/claim $FOCUS_ARG $TNF_FOCUS_SUITES $SKIP_ARG $TNF_SKIP_SUITES "$@" diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index e139d097e..2f5c7b09e 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -73,7 +73,8 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { //nolint:gocritic,funlen // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, testType string) { - ginkgo.It(fmt.Sprintf("Running Pod test : %s", testCmd.Name), func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHostResourceIdentifier) + "-" + testCmd.Name + ginkgo.It(testID, func() { context := common.GetContext() for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name @@ -132,11 +133,12 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, func testNamespace(env *config.TestEnvironment) { ginkgo.When("test deployment namespace", func() { - ginkgo.It("Should not be 'default' and should not begin with 'openshift-'", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNamespaceBestPracticesIdentifier) + ginkgo.It(testID, func() { for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) + ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s, should not be 'default' or begin with openshift-", podNamespace, podName)) defer results.RecordResult(identifiers.TestNamespaceBestPracticesIdentifier) gomega.Expect(podNamespace).To(gomega.Not(gomega.Equal("default"))) gomega.Expect(podNamespace).To(gomega.Not(gomega.HavePrefix("openshift-"))) @@ -152,7 +154,9 @@ func testRoles(env *config.TestEnvironment) { } func testServiceAccount(env *config.TestEnvironment) { - ginkgo.It("Should have a valid ServiceAccount name", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodServiceAccountBestPracticesIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Should have a valid ServiceAccount name") for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace @@ -172,7 +176,9 @@ func testServiceAccount(env *config.TestEnvironment) { } func testRoleBindings(env *config.TestEnvironment) { - ginkgo.It("Should not have RoleBinding in other namespaces", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRoleBindingsBestPracticesIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Should not have RoleBinding in other namespaces") for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace @@ -197,7 +203,9 @@ func testRoleBindings(env *config.TestEnvironment) { } func testClusterRoleBindings(env *config.TestEnvironment) { - ginkgo.It("Should not have ClusterRoleBindings", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRoleBindingsBestPracticesIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Should not have ClusterRoleBindings") for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace diff --git a/test-network-function/certification/suite.go b/test-network-function/certification/suite.go index 7a3ba500a..7eb0dc5a4 100644 --- a/test-network-function/certification/suite.go +++ b/test-network-function/certification/suite.go @@ -41,37 +41,38 @@ var certAPIClient api.CertAPIClient var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AffiliatedCertTestKey) { - return + testContainerCertificationStatus() + testOperatorCertificationStatus() } - testContainerCertificationStatus() - testOperatorCertificationStatus() }) func testContainerCertificationStatus() { // Query API for certification status of listed containers - ginkgo.When("getting certification status", func() { - ginkgo.It("get certification status", func() { - defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) - env := configpkg.GetTestEnvironment() - cnfsToQuery := env.Config.CertifiedContainerInfo - if len(cnfsToQuery) > 0 { - certAPIClient = api.NewHTTPClient() - for _, cnf := range cnfsToQuery { - cnf := cnf // pin - // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. - ginkgo.By(fmt.Sprintf("container %s/%s should eventually be verified as certified", cnf.Repository, cnf.Name)) - gomega.Eventually(func() bool { - isCertified := certAPIClient.IsContainerCertified(cnf.Repository, cnf.Name) - return isCertified - }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) - } + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestContainerIsCertifiedIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("getting certification status") + defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) + env := configpkg.GetTestEnvironment() + cnfsToQuery := env.Config.CertifiedContainerInfo + if len(cnfsToQuery) > 0 { + certAPIClient = api.NewHTTPClient() + for _, cnf := range cnfsToQuery { + cnf := cnf // pin + // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. + ginkgo.By(fmt.Sprintf("container %s/%s should eventually be verified as certified", cnf.Repository, cnf.Name)) + gomega.Eventually(func() bool { + isCertified := certAPIClient.IsContainerCertified(cnf.Repository, cnf.Name) + return isCertified + }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) } - }) + } }) } func testOperatorCertificationStatus() { - ginkgo.It("Verify operator as certified", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsCertifiedIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Verify operator as certified") defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) operatorsToQuery := configpkg.GetTestEnvironment().Config.CertifiedOperatorInfo if len(operatorsToQuery) > 0 { diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index 44fa82d54..7dca0dbbf 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -63,8 +63,8 @@ func IsMinikube() bool { return b } -// NonIntrusive is for skipping tests that would impact the CNF or test environment in an intrusive way -func NonIntrusive() bool { +// Intrusive is for running tests that can impact the CNF or test environment in an intrusive way +func Intrusive() bool { b, _ := strconv.ParseBool(os.Getenv("TNF_NON_INTRUSIVE_ONLY")) - return b + return !b } diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 361ee89dd..39f314af8 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -11,12 +11,14 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" + ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodedebug" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/test-network-function/test-network-function/pkg/tnf/testcases" ) const ( @@ -61,47 +63,56 @@ var ( ) var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { - ginkgo.When("a cluster is set up and installed with OpenShift", func() { - ginkgo.It("should report all available nodeSummary", func() { - defer results.RecordResult(identifiers.TestExtractNodeInformationIdentifier) - context := common.GetContext() - - test, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(jsonParseResult).ToNot(gomega.BeNil()) - gomega.Expect(jsonParseResult.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - - result, err := tester.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).To(gomega.Equal(tnf.SUCCESS)) - - genericTest := (*test).(*generic.Generic) - gomega.Expect(genericTest).ToNot(gomega.BeNil()) - matches := genericTest.Matches - gomega.Expect(len(matches)).To(gomega.Equal(1)) - match := genericTest.GetMatches()[0] - err = json.Unmarshal([]byte(match.Match), &nodeSummary) - gomega.Expect(err).To(gomega.BeNil()) + if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.DiagnosticTestKey) { + ginkgo.When("a cluster is set up and installed with OpenShift", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestExtractNodeInformationIdentifier) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestExtractNodeInformationIdentifier) + context := common.GetContext() + + test, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(jsonParseResult).ToNot(gomega.BeNil()) + gomega.Expect(jsonParseResult.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + + tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + result, err := tester.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).To(gomega.Equal(tnf.SUCCESS)) + + genericTest := (*test).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + match := genericTest.GetMatches()[0] + err = json.Unmarshal([]byte(match.Match), &nodeSummary) + gomega.Expect(err).To(gomega.BeNil()) + }) + ginkgo.By("should report all CNI plugins") + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestListCniPluginsIdentifier) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestListCniPluginsIdentifier) + testCniPlugins() + }) + ginkgo.By("should report nodes HW info") + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestNodesHwInfoIdentifier) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestNodesHwInfoIdentifier) + testNodesHwInfo() + }) + ginkgo.By("should report cluster CSI driver info") + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestClusterCsiInfoIdentifier) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestClusterCsiInfoIdentifier) + listClusterCSIInfo() + }) }) - ginkgo.It("should report all CNI plugins", func() { - defer results.RecordResult(identifiers.TestListCniPluginsIdentifier) - testCniPlugins() - }) - ginkgo.It("should report nodes HW info", func() { - defer results.RecordResult(identifiers.TestNodesHwInfoIdentifier) - testNodesHwInfo() - }) - ginkgo.It("should report cluster CSI driver info", func() { - defer results.RecordResult(identifiers.TestClusterCsiInfoIdentifier) - listClusterCSIInfo() - }) - }) + } }) // CniPlugin holds info about a CNI plugin diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 86d720c22..8a307368c 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -19,6 +19,7 @@ package identifiers import ( "fmt" + "strings" "github.com/test-network-function/test-network-function-claim/pkg/claim" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -205,6 +206,12 @@ func formDescription(identifier claim.Identifier, description string) string { return fmt.Sprintf("%s %s", identifier.Url, description) } +// XformToGinkgoItIdentifier transform the claim.Identifier into a test Id that can be used to skip +// specific tests +func XformToGinkgoItIdentifier(identifier claim.Identifier) string { + return strings.ReplaceAll(strings.TrimPrefix(identifier.Url, url+"/"), "/", "-") +} + // Catalog is the JUnit testcase catalog of tests. var Catalog = map[claim.Identifier]TestCaseDescription{ diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 5e7d255dc..ffd7c63b4 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -86,11 +86,11 @@ var drainTimeout = time.Duration(drainTimeoutMinutes) * time.Minute // All actual test code belongs below here. Utilities belong above. // var _ = ginkgo.Describe(common.LifecycleTestKey, func() { - env := config.GetTestEnvironment() - ginkgo.BeforeEach(func() { - env.LoadAndRefresh() - }) if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.LifecycleTestKey) { + env := config.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + }) testNodeSelector(env) @@ -99,7 +99,8 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { testShutdown(env) testPodAntiAffinity(env) - if !common.NonIntrusive() { + + if common.Intrusive() { testPodsRecreation(env) testScaling(env) @@ -168,7 +169,9 @@ func runScalingTest(namespace, deploymentName string, replicaCount int) { } func testScaling(env *config.TestEnvironment) { - ginkgo.It("Testing deployment scaling", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestScalingIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Testing deployment scaling") defer results.RecordResult(identifiers.TestScalingIdentifier) namespaceDeploymentsBackup := make(map[string]dp.DeploymentMap) @@ -214,7 +217,9 @@ func testScaling(env *config.TestEnvironment) { } func testNodeSelector(env *config.TestEnvironment) { - ginkgo.It("Testing pod nodeSelector", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodNodeSelectorAndAffinityBestPractices) + ginkgo.It(testID, func() { + ginkgo.By("Testing pod nodeSelector") context := common.GetContext() for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name @@ -237,43 +242,43 @@ func testNodeSelector(env *config.TestEnvironment) { } func testGracePeriod(env *config.TestEnvironment) { - ginkgo.When("Test terminationGracePeriod ", func() { - ginkgo.It("Testing pod terminationGracePeriod", func() { - context := common.GetContext() - for _, podUnderTest := range env.PodsUnderTest { - podName := podUnderTest.Name - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Testing pod terminationGracePeriod %s %s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestNonDefaultGracePeriodIdentifier) - infoWriter := tnf.CreateTestExtraInfoWriter() - tester := graceperiod.NewGracePeriod(common.DefaultTimeout, podName, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - gracePeriod := tester.GetGracePeriod() - if gracePeriod == defaultTerminationGracePeriod { - msg := fmt.Sprintf("%s %s has terminationGracePeriod set to %d, you might want to change it", podNamespace, podName, defaultTerminationGracePeriod) - log.Warn(msg) - infoWriter(msg) - } + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonDefaultGracePeriodIdentifier) + ginkgo.It(testID+" ", func() { + ginkgo.By("Test terminationGracePeriod") + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("Testing pod terminationGracePeriod %s %s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestNonDefaultGracePeriodIdentifier) + infoWriter := tnf.CreateTestExtraInfoWriter() + tester := graceperiod.NewGracePeriod(common.DefaultTimeout, podName, podNamespace) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + gracePeriod := tester.GetGracePeriod() + if gracePeriod == defaultTerminationGracePeriod { + msg := fmt.Sprintf("%s %s has terminationGracePeriod set to %d, you might want to change it", podNamespace, podName, defaultTerminationGracePeriod) + log.Warn(msg) + infoWriter(msg) } - }) + } }) } func testShutdown(env *config.TestEnvironment) { - ginkgo.When("Testing PUTs are configured with pre-stop lifecycle", func() { - ginkgo.It("should have pre-stop configured", func() { - for _, podUnderTest := range env.PodsUnderTest { - podName := podUnderTest.Name - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("should have pre-stop configured %s/%s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestShudtownIdentifier) - shutdownTest(podNamespace, podName) - } - }) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestShudtownIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Testing PUTs are configured with pre-stop lifecycle") + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("should have pre-stop configured %s/%s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestShudtownIdentifier) + shutdownTest(podNamespace, podName) + } }) } @@ -305,8 +310,10 @@ func testPodsRecreation(env *config.TestEnvironment) { var notReadyDeployments []string nodesNames := make(map[string]node) namespaces := make(map[string]bool) - ginkgo.It("Testing node draining effect of deployment", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRecreationIdentifier) + ginkgo.It(testID, func() { env.SetNeedsRefresh() + ginkgo.By("Testing node draining effect of deployment") for _, podUnderTest := range env.PodsUnderTest { podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("test deployment in namespace %s", podNamespace)) @@ -426,7 +433,9 @@ func uncordonNode(node string) { func testPodAntiAffinity(env *config.TestEnvironment) { var deployments dp.DeploymentMap ginkgo.When("CNF is designed in high availability mode ", func() { - ginkgo.It("Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodHighAvailabilityBestPractices) + ginkgo.It(testID, func() { + ginkgo.By("Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment") for _, podUnderTest := range env.PodsUnderTest { podNamespace := podUnderTest.Namespace defer results.RecordResult(identifiers.TestPodHighAvailabilityBestPractices) @@ -482,21 +491,21 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { } func testOwner(env *config.TestEnvironment) { - ginkgo.When("Testing owners of CNF pod", func() { - ginkgo.It("Should be only ReplicaSet", func() { - context := common.GetContext() - for _, podUnderTest := range env.PodsUnderTest { - podName := podUnderTest.Name - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Should be ReplicaSet %s %s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestPodDeploymentBestPracticesIdentifier) - tester := owners.NewOwners(common.DefaultTimeout, podNamespace, podName) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - } - }) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodDeploymentBestPracticesIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Testing owners of CNF pod, should be replicas Set") + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + podName := podUnderTest.Name + podNamespace := podUnderTest.Namespace + ginkgo.By(fmt.Sprintf("Should be ReplicaSet %s %s", podNamespace, podName)) + defer results.RecordResult(identifiers.TestPodDeploymentBestPracticesIdentifier) + tester := owners.NewOwners(common.DefaultTimeout, podNamespace, podName) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + } }) } diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 9a0deed53..4fe2830fb 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -47,11 +47,11 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(common.NetworkingTestKey, func() { - env := config.GetTestEnvironment() - ginkgo.BeforeEach(func() { - env.LoadAndRefresh() - }) if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.NetworkingTestKey) { + env := config.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + }) ginkgo.Context("Both Pods are on the Default network", func() { // for each container under test, ensure bidirectional ICMP traffic between the container and the orchestrator. testDefaultNetworkConnectivity(env, defaultNumPings) @@ -69,7 +69,8 @@ var _ = ginkgo.Describe(common.NetworkingTestKey, func() { func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { - ginkgo.It("should reply to ping", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestICMPv4ConnectivityIdentifier) + ginkgo.It(testID, func() { for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue @@ -92,7 +93,8 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { - ginkgo.It("should reply to ping", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestICMPv4ConnectivityIdentifier) + ginkgo.It(testID, func() { for _, cut := range env.ContainersUnderTest { for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { testOrchestrator := env.TestOrchestrator @@ -120,7 +122,8 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count } func testNodePort(env *config.TestEnvironment) { - ginkgo.It("Should not have services of type NodePort", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestServicesDoNotUseNodeportsIdentifier) + ginkgo.It(testID, func() { for _, podUnderTest := range env.PodsUnderTest { defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) context := common.GetContext() diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 330b855d8..a306484d7 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -55,14 +55,13 @@ var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { }) func testLogging(env *config.TestEnvironment) { - ginkgo.When("Testing PUT is emitting logs to stdout/stderr", func() { - ginkgo.It("should return at least one line of log", func() { - for _, cut := range env.ContainersUnderTest { - ginkgo.By(fmt.Sprintf("Test container: %+v", cut.ContainerIdentifier)) - defer results.RecordResult(identifiers.TestLoggingIdentifier) - loggingTest(cut.ContainerIdentifier) - } - }) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestLoggingIdentifier) + ginkgo.It(testID, func() { + for _, cut := range env.ContainersUnderTest { + ginkgo.By(fmt.Sprintf("Test container: %+v. should emit at least one line of log to stderr/stdout", cut.ContainerIdentifier)) + defer results.RecordResult(identifiers.TestLoggingIdentifier) + loggingTest(cut.ContainerIdentifier) + } }) } func loggingTest(c configsections.ContainerIdentifier) { diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 7502b46cd..59e86addd 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -95,7 +95,8 @@ var _ = ginkgo.Describe(testSpecName, func() { // testOperatorsAreInstalledViaOLM ensures all configured operators have a proper OLM subscription. func testOperatorsAreInstalledViaOLM() { - ginkgo.It("operator-olm-installed", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsInstalledViaOLMIdentifier) + ginkgo.It(testID, func() { _, operatorsInTest := getConfig() for _, operatorInTest := range operatorsInTest { defer results.RecordResult(identifiers.TestOperatorIsInstalledViaOLMIdentifier) @@ -135,7 +136,8 @@ func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsection } func itRunsTestsOnOperator() { - ginkgo.It("operator-certification", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsCertifiedIdentifier) + ginkgo.It(testID, func() { operatorsToQuery, operatorsInTest := getConfig() if len(operatorsToQuery) > 0 { certAPIClient := api.NewHTTPClient() @@ -189,18 +191,17 @@ func agrName(operatorName, subName, testName string) string { //nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. func runTestsOnOperator(args []interface{}, name, namespace string, testCmd testcases.BaseTestCase) { - ginkgo.When(fmt.Sprintf("under test is: %s/%s ", namespace, name), func() { - ginkgo.It(fmt.Sprintf("tests for: %s", testCmd.Name), func() { - defer results.RecordResult(identifiers.TestOperatorInstallStatusIdentifier) - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") - opInTest := operator.NewOperator(cmdArgs, name, namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, defaultTimeout) - gomega.Expect(opInTest).ToNot(gomega.BeNil()) - test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - }) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorInstallStatusIdentifier) + "-" + testCmd.Name + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestOperatorInstallStatusIdentifier) + cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") + opInTest := operator.NewOperator(cmdArgs, name, namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, defaultTimeout) + gomega.Expect(opInTest).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) }) } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 4d11c41e2..5ce21de5d 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -86,7 +86,9 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { // testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. func testIsRedHatRelease(env *config.TestEnvironment) { - ginkgo.It("Should report a proper Red Hat version", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("should report a proper Red Hat version") defer results.RecordResult(identifiers.TestIsRedHatReleaseIdentifier) for _, cut := range env.ContainersUnderTest { testContainerIsRedHatRelease(cut) @@ -110,7 +112,8 @@ func testContainerIsRedHatRelease(cut *config.Container) { // testContainersFsDiff test that all CUT didn't install new packages are starting func testContainersFsDiff(env *config.TestEnvironment) { - ginkgo.It("platform-fsdiff", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredBaseImageIdentifier) + ginkgo.It(testID, func() { fsDiffContainer := env.FsDiffMasterContainer if fsDiffContainer != nil { for _, cut := range env.ContainersUnderTest { @@ -239,7 +242,8 @@ func getSysctlConfigArgs(context *interactive.Context, nodeName string) map[stri } func testBootParams(env *config.TestEnvironment) { - ginkgo.It("platform-boot-param", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredStartupBootParamsIdentifier) + ginkgo.It(testID, func() { context := common.GetContext() for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() @@ -296,39 +300,38 @@ func testTainted() { return } var nodeNames []string - ginkgo.When("Testing tainted nodes in cluster", func() { - ginkgo.It("Should return list of node names", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) + ginkgo.It(testID, func() { + ginkgo.By("Testing tainted nodes in cluster") + ginkgo.By("Should return list of node names") + context := common.GetContext() + tester := nodenames.NewNodeNames(common.DefaultTimeout, nil) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + nodeNames = tester.GetNodeNames() + gomega.Expect(nodeNames).NotTo(gomega.BeNil()) + ginkgo.By("Should not have tainted nodes") + defer results.RecordResult(identifiers.TestNonTaintedNodeKernelsIdentifier) + if len(nodeNames) == 0 { + ginkgo.Skip("Can't test tainted nodes when list of nodes is empty. Please check previous tests.") + } + var taintedNodes []string + for _, node := range nodeNames { context := common.GetContext() - tester := nodenames.NewNodeNames(common.DefaultTimeout, nil) + tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(testResult).NotTo(gomega.Equal(tnf.ERROR)) gomega.Expect(err).To(gomega.BeNil()) - nodeNames = tester.GetNodeNames() - gomega.Expect(nodeNames).NotTo(gomega.BeNil()) - }) - - ginkgo.It("Should not have tainted nodes", func() { - defer results.RecordResult(identifiers.TestNonTaintedNodeKernelsIdentifier) - if len(nodeNames) == 0 { - ginkgo.Skip("Can't test tainted nodes when list of nodes is empty. Please check previous tests.") - } - var taintedNodes []string - for _, node := range nodeNames { - context := common.GetContext() - tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).NotTo(gomega.Equal(tnf.ERROR)) - gomega.Expect(err).To(gomega.BeNil()) - if testResult == tnf.FAILURE { - taintedNodes = append(taintedNodes, node) - } + if testResult == tnf.FAILURE { + taintedNodes = append(taintedNodes, node) } - gomega.Expect(taintedNodes).To(gomega.BeNil()) - }) + } + gomega.Expect(taintedNodes).To(gomega.BeNil()) }) } @@ -338,44 +341,44 @@ func testHugepages() { } var nodeNames []string var clusterHugepages, clusterHugepagesz int - ginkgo.When("Testing worker nodes' hugepages configuration", func() { - ginkgo.It("Should return list of worker node names", func() { - context := common.GetContext() - tester := nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{"node-role.kubernetes.io/worker": nil}) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - nodeNames = tester.GetNodeNames() - gomega.Expect(nodeNames).NotTo(gomega.BeNil()) - }) - ginkgo.It("Should return cluster's hugepages configuration", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) + ginkgo.By("Should return list of worker node names") + context := common.GetContext() + tester := nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{"node-role.kubernetes.io/worker": nil}) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + nodeNames = tester.GetNodeNames() + gomega.Expect(nodeNames).NotTo(gomega.BeNil()) + + ginkgo.By("Should return cluster's hugepages configuration") + context = common.GetContext() + hugepageTester := hugepages.NewHugepages(common.DefaultTimeout) + test, err = tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err = test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + clusterHugepages = hugepageTester.GetHugepages() + clusterHugepagesz = hugepageTester.GetHugepagesz() + + ginkgo.By("Should have same configuration as cluster") + var badNodes []string + for _, node := range nodeNames { context := common.GetContext() - tester := hugepages.NewHugepages(common.DefaultTimeout) + tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node, clusterHugepagesz, clusterHugepages) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) - clusterHugepages = tester.GetHugepages() - clusterHugepagesz = tester.GetHugepagesz() - }) - ginkgo.It("Should have same configuration as cluster", func() { - defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) - var badNodes []string - for _, node := range nodeNames { - context := common.GetContext() - tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node, clusterHugepagesz, clusterHugepages) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - if testResult != tnf.SUCCESS { - badNodes = append(badNodes, node) - } + if testResult != tnf.SUCCESS { + badNodes = append(badNodes, node) } - gomega.Expect(badNodes).To(gomega.BeNil()) - }) + } + gomega.Expect(badNodes).To(gomega.BeNil()) }) } From 79dbf9984c3bd80c40f1afa890fbaa70a639371e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 09:46:50 -0400 Subject: [PATCH 019/344] Bump github.com/onsi/gomega from 1.15.0 to 1.16.0 (#316) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Salaheddine Hamadi --- go.mod | 3 +-- go.sum | 14 ++------------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 31647491d..525a37f67 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,12 @@ require ( github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f github.com/kr/pretty v0.2.1 // indirect github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.15.0 + github.com/onsi/gomega v1.16.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect google.golang.org/grpc v1.39.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index ab9ec6ae0..ce7b7ea9e 100644 --- a/go.sum +++ b/go.sum @@ -41,7 +41,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/a-h/generate v0.0.0-20190312091541-e59c34d33fb3/go.mod h1:traiLYQ0YD7qUMCdjo6/jSaJRPHXniX4HVs+PhEhYpc= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -56,7 +55,6 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -87,7 +85,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -225,8 +222,8 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -245,7 +242,6 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= @@ -254,7 +250,6 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -324,7 +319,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -336,7 +330,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -515,7 +508,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -615,8 +607,6 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From b1747471c7f5ae4289606dddeb14e78bdd3065fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 15:08:22 -0400 Subject: [PATCH 020/344] Bump google.golang.org/grpc from 1.39.0 to 1.40.0 (#315) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.39.0 to 1.40.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.39.0...v1.40.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 525a37f67..fca27800e 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,6 @@ require ( github.com/test-network-function/test-network-function-claim v1.0.3 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - google.golang.org/grpc v1.39.0 + google.golang.org/grpc v1.40.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index ce7b7ea9e..ea9b0a506 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/a-h/generate v0.0.0-20190312091541-e59c34d33fb3/go.mod h1:traiLYQ0YD7qUMCdjo6/jSaJRPHXniX4HVs+PhEhYpc= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -55,6 +56,7 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -242,6 +244,7 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= @@ -605,8 +608,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 6d7ec547a11051291f40ba27aa603d3b224adadf Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 25 Aug 2021 21:28:37 +0200 Subject: [PATCH 021/344] Fixed operator TS config loading. (#318) * Fixed operator TS config loading. The certifying part is always done, but the operators test is skipped when running in Minikube. * Fixed linter errors. * Fixed certification TS config loading. The certification TS was not loading the yaml file so it always had an empty list of containers to check. --- .github/workflows/pre-main.yaml | 2 +- test-network-function/certification/suite.go | 15 +++++++++++---- test-network-function/operator/suite.go | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 52b4df3e1..0d880233d 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -123,7 +123,7 @@ jobs: run: ./run-cnf-suites.sh --focus diagnostic - name: 'Test: Run test suites' - run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification + run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification operator # Perform smoke tests using a TNF container. diff --git a/test-network-function/certification/suite.go b/test-network-function/certification/suite.go index 7eb0dc5a4..5fb341160 100644 --- a/test-network-function/certification/suite.go +++ b/test-network-function/certification/suite.go @@ -41,6 +41,11 @@ var certAPIClient api.CertAPIClient var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AffiliatedCertTestKey) { + env := configpkg.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + }) + testContainerCertificationStatus() testOperatorCertificationStatus() } @@ -50,10 +55,12 @@ func testContainerCertificationStatus() { // Query API for certification status of listed containers testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestContainerIsCertifiedIdentifier) ginkgo.It(testID, func() { - ginkgo.By("getting certification status") - defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) env := configpkg.GetTestEnvironment() cnfsToQuery := env.Config.CertifiedContainerInfo + + ginkgo.By(fmt.Sprintf("Getting certification status. Number of containers to check: %d", len(cnfsToQuery))) + defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) + if len(cnfsToQuery) > 0 { certAPIClient = api.NewHTTPClient() for _, cnf := range cnfsToQuery { @@ -72,9 +79,9 @@ func testContainerCertificationStatus() { func testOperatorCertificationStatus() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsCertifiedIdentifier) ginkgo.It(testID, func() { - ginkgo.By("Verify operator as certified") - defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) operatorsToQuery := configpkg.GetTestEnvironment().Config.CertifiedOperatorInfo + ginkgo.By(fmt.Sprintf("Verify operator as certified. Number of operators to check: %d", len(operatorsToQuery))) + defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) if len(operatorsToQuery) > 0 { certAPIClient := api.NewHTTPClient() for _, certified := range operatorsToQuery { diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 59e86addd..d3bd4be8c 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -24,12 +24,14 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/internal/api" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" @@ -75,6 +77,11 @@ var ( var _ = ginkgo.Describe(testSpecName, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testSpecName) { + env := config.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + }) + defer ginkgo.GinkgoRecover() ginkgo.When("a local shell is spawned", func() { goExpectSpawner := interactive.NewGoExpectSpawner() @@ -152,7 +159,14 @@ func itRunsTestsOnOperator() { }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) } } - gomega.Expect(operatorsInTest).ToNot(gomega.BeNil()) + if common.IsMinikube() { + log.Info("Minikube detected: skipping operators test.") + return + } + if len(operatorsInTest) == 0 { + ginkgo.Fail("No operators to test were discovered.") + } + for _, op := range operatorsInTest { // TODO: Gather facts for operator for _, testType := range op.Tests { From 6d9d95a9470eefdb85f1ecbd9ee595781a7fe69f Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 26 Aug 2021 11:09:29 -0500 Subject: [PATCH 022/344] Do not pass test cases if no container/pod under test present (#319) * Do not pass test cases if no container/pod under test present * Making it simpler based on comments --- test-network-function/accesscontrol/suite.go | 2 ++ test-network-function/lifecycle/suite.go | 3 +++ test-network-function/networking/suite.go | 2 ++ test-network-function/observability/suite.go | 2 ++ test-network-function/platform/suite.go | 2 ++ 5 files changed, 11 insertions(+) diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 2f5c7b09e..800af5ff0 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -42,6 +42,8 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) testNamespace(env) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index ffd7c63b4..93e2c7ba7 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -90,6 +90,9 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) + }) testNodeSelector(env) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 4fe2830fb..a92bd513b 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -51,6 +51,8 @@ var _ = ginkgo.Describe(common.NetworkingTestKey, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) ginkgo.Context("Both Pods are on the Default network", func() { // for each container under test, ensure bidirectional ICMP traffic between the container and the orchestrator. diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index a306484d7..bc4349036 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -49,6 +49,8 @@ var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) testLogging(env) } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 5ce21de5d..422eba7f0 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -60,6 +60,8 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) ginkgo.Context("Container does not have additional packages installed", func() { // use this boolean to turn off tests that require OS packages From 094c3fc6eb4e658ee6c3a3d245cdb97a634a4f27 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 26 Aug 2021 12:40:02 -0400 Subject: [PATCH 023/344] Fix hugepage test (#321) Co-authored-by: wying3 <81822258+wying3@users.noreply.github.com> --- run-cnf-suites.sh | 2 +- test-network-function/platform/suite.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index ba8a97837..ed2cf9405 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -71,5 +71,5 @@ fi echo "Running with focus '$FOCUS'" echo "Running with skip '$SKIP'" echo "Report will be output to '$OUTPUT_LOC'" - +echo "ginkgo arguments '${GINKGO_ARGS}'" cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" -ginkgo.skip="$SKIP" ${GINKGO_ARGS} diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 422eba7f0..3cf96d3b1 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -360,7 +360,7 @@ func testHugepages() { ginkgo.By("Should return cluster's hugepages configuration") context = common.GetContext() hugepageTester := hugepages.NewHugepages(common.DefaultTimeout) - test, err = tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{tester}, context.GetErrorChannel()) + test, err = tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err = test.Run() gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) @@ -369,6 +369,7 @@ func testHugepages() { clusterHugepagesz = hugepageTester.GetHugepagesz() ginkgo.By("Should have same configuration as cluster") + ginkgo.By(fmt.Sprintf("cluster is configured with clusterHugepages=%d ; clusterHugepagesz=%d", clusterHugepages, clusterHugepagesz)) var badNodes []string for _, node := range nodeNames { context := common.GetContext() @@ -379,6 +380,7 @@ func testHugepages() { gomega.Expect(err).To(gomega.BeNil()) if testResult != tnf.SUCCESS { badNodes = append(badNodes, node) + ginkgo.By(fmt.Sprintf("node=%s hugepage config does not match machineconfig", node)) } } gomega.Expect(badNodes).To(gomega.BeNil()) From 0822acca0103a445f79400f9f5378aa6895b8598 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 26 Aug 2021 13:19:19 -0500 Subject: [PATCH 024/344] Configure Loglevel for logrus (#320) * Configure Loglevel for logrus * printing better logs incase no loglevel is passed * listing valid values * Adding container support * Resolving comments * resolve comment in README Co-authored-by: Salaheddine Hamadi --- README.md | 2 + pkg/config/autodiscover/pod_info.go | 4 +- script/run-container.sh | 1 + test-network-function/common/env.go | 62 +++++++++++++++++++++++++++++ test-network-function/suite_test.go | 2 + 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f0ff22e8..31eb844f0 100644 --- a/README.md +++ b/README.md @@ -463,6 +463,8 @@ operator /Users/$USER/cnf-cert/test-network-function/test-network-function/operator/suite.go:152 ``` +## Log level +The optional LOG_LEVEL environment variable sets the log level. Defaults to "info" if not set. Valid values are: trace, debug, info, warn, error, fatal, panic. ## Grading Tool ### Overview diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index ba01dcedd..c2645d873 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -157,7 +157,9 @@ func GetPodsByLabel(label configsections.Label) (*PodList, error) { if err != nil { return nil, err } - + log.Debug("JSON output for all pods labeled with: ", label) + log.Debug("Command: ", cmd) + log.Debug(string(out)) var podList PodList err = json.Unmarshal(out, &podList) if err != nil { diff --git a/script/run-container.sh b/script/run-container.sh index 113b1dc78..2a521bf4a 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -106,6 +106,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_MINIKUBE_ONLY=$CONTAINER_TNF_MINIKUBE_ONLY \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ + -e LOG_LEVEL=$LOGLEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ $TNF_CMD $OUTPUT_ARG $CONTAINER_TNF_DIR/claim $FOCUS_ARG $TNF_FOCUS_SUITES $SKIP_ARG $TNF_SKIP_SUITES "$@" diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index 7dca0dbbf..bd836e89d 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -17,12 +17,15 @@ package common import ( + "errors" "os" "path" "strconv" + "strings" "time" "github.com/onsi/gomega" + "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" ) @@ -38,6 +41,20 @@ var ( schemaPath = path.Join("schemas", "generic-test.schema.json") ) +const ( + logLevelTraceString = "trace" + logLevelDebugString = "debug" + logLevelInfoString = "info" + logLevelWarnString = "warn" + logLevelErrorString = "error" + logLevelFatalString = "fatal" + logLevelPanicString = "panic" + logLevelEmptyString = "" + logLevelDefault = logrus.InfoLevel + errorEmpty = "empty" + errorInvalid = "invalid" +) + // DefaultTimeout for creating new interactive sessions (oc, ssh, tty) var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second @@ -68,3 +85,48 @@ func Intrusive() bool { b, _ := strconv.ParseBool(os.Getenv("TNF_NON_INTRUSIVE_ONLY")) return !b } + +// logLevel retrieves the LOG_LEVEL environement vaiable +func logLevel() string { + return os.Getenv("LOG_LEVEL") +} + +// stringToLogLevel converts a string to a log logrus level +func stringToLogLevel(logLevelString string) (logrus.Level, error) { + logLevelString = strings.ToLower(logLevelString) + switch logLevelString { + case logLevelTraceString: + return logrus.TraceLevel, nil + case logLevelDebugString: + return logrus.DebugLevel, nil + case logLevelInfoString: + return logrus.InfoLevel, nil + case logLevelWarnString: + return logrus.WarnLevel, nil + case logLevelErrorString: + return logrus.ErrorLevel, nil + case logLevelFatalString: + return logrus.FatalLevel, nil + case logLevelPanicString: + return logrus.PanicLevel, nil + case logLevelEmptyString: + return logLevelDefault, errors.New(errorEmpty) + } + return logLevelDefault, errors.New(errorInvalid) +} + +// SetLogLevel sets the log level for logrus based on the "LOG_LEVEL" environment variable +func SetLogLevel() { + var aLogLevel, err = stringToLogLevel(logLevel()) + + if err != nil { + if err.Error() == errorInvalid { + logrus.Info("LOG_LEVEL environment set with an invalid value, defaulting to", logLevelDefault, ".\n Valid values are: trace, debug, info, warn, error, fatal, panic") + } + if err.Error() == errorEmpty { + logrus.Info("LOG_LEVEL environment variable not set, defaulting to: ", logLevelDefault, ".\n Valid values are: trace, debug, info, warn, error, fatal, panic") + } + } + logrus.Info("Log level set to:", aLogLevel) + logrus.SetLevel(aLogLevel) +} diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 02294d368..f6855671c 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -37,6 +37,7 @@ import ( _ "github.com/test-network-function/test-network-function/test-network-function/accesscontrol" _ "github.com/test-network-function/test-network-function/test-network-function/certification" + "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/diagnostic" _ "github.com/test-network-function/test-network-function/test-network-function/generic" _ "github.com/test-network-function/test-network-function/test-network-function/lifecycle" @@ -110,6 +111,7 @@ func TestTest(t *testing.T) { // set up input flags and register failure handlers. flag.Parse() gomega.RegisterFailHandler(ginkgo.Fail) + common.SetLogLevel() // Initialize the claim with the start time, tnf version, etc. claimRoot := createClaimRoot() From d6a028c2b0e94be184c10825c297e7e237623c34 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 26 Aug 2021 20:54:19 -0500 Subject: [PATCH 025/344] Adding version log (#323) * Adding version log * Adding Major version and lint --- Makefile | 5 +++-- test-network-function/suite_test.go | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ed5d6d105..794d2e009 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ else endif COMMON_GO_ARGS=-race +GIT_COMMIT=$(shell git rev-list -1 HEAD) # Run the unit tests and build all binaries build: @@ -95,11 +96,11 @@ build-catalog-md: # build the CNF test binary build-cnf-tests: - PATH=${PATH}:${GOBIN} ginkgo build ./test-network-function + PATH=${PATH}:${GOBIN} ginkgo build -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT}" ./test-network-function make build-catalog-md build-cnf-tests-debug: - PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-extldflags '-z relro -z now'" ./test-network-function + PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -extldflags '-z relro -z now'" ./test-network-function make build-catalog-md # run all CNF tests diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index f6855671c..b45197ac4 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -49,6 +49,7 @@ import ( ) const ( + programVersion = "3.0" claimFileName = "claim.json" claimFilePermissions = 0644 claimPathFlagKey = "claimloc" @@ -68,6 +69,7 @@ const ( var ( claimPath *string junitPath *string + GitCommit string ) func init() { @@ -106,11 +108,12 @@ func loadJUnitXMLIntoMap(result map[string]interface{}, junitFilename, key strin } } -// TestTest invokes the CNF Certification Test Suite. +//nolint:funlen // TestTest invokes the CNF Certification Test Suite. func TestTest(t *testing.T) { // set up input flags and register failure handlers. flag.Parse() gomega.RegisterFailHandler(ginkgo.Fail) + log.Info("Version: ", programVersion, " ( ", GitCommit, " )") common.SetLogLevel() // Initialize the claim with the start time, tnf version, etc. From 16d2508525d88943b676890d04eaeec8a67ad8c8 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 26 Aug 2021 23:39:11 -0400 Subject: [PATCH 026/344] readme update : debug container deployment (#322) * readme update : debug container deployment * fix review * fix PR review --- README.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 31eb844f0..12b101a3d 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The corresponding label prefix is: test-network-function.com/generic: target ``` -When labelling a pod to be discovered and tested, the discoered pods are **in addition to** the ones +When labelling a pod to be discovered and tested, the discovered pods are **in addition to** the ones explicitly configured in the testTarget sections of the config file. ### testTarget @@ -140,19 +140,33 @@ For more information on the test suites, refer to [the cnf-features-deploy repos ## Running the tests with in a prebuild container -A ready to run container is available at this repository: [quay.io](https://quay.io/repository/testnetworkfunction/test-network-function) +### Pulling test image +An image is built and is available at this repository: [quay.io](https://quay.io/repository/testnetworkfunction/test-network-function) +The image can be pulled using : +```shell script +docker pull quay.io/testnetworkfunction/test-network-function +``` +### Check cluster resources +Some tests suites such as platform-alteration require node access to get node configuration like hugepage. +In order to get the required information, the test suite does not ssh into nodes, but instead rely on [oc debug tools ](https://docs.openshift.com/container-platform/3.7/cli_reference/basic_cli_operations.html#debug). This tool makes it easier to fetch information from nodes and also to debug running pods. + +In short, oc debug tool will launch a new container ending with "-debug" suffix, the container will be destroyed once the debug session is done. To be able to create the debug pod, the cluster should have enough resources, otherwise those tests would fail. -To pull the latest container and run the tests you use the following command. There are several required arguments: +**Note:** +It's recommended to clean up disk space and make sure there's enough resources to deploy another container image in every node before starting the tests. +### Run the tests +``./run-tnf-container.sh`` script is used to launch the tests. + +There are several required arguments: * `-t` gives the local directory that contains tnf config files set up for the test. * `-o` gives the local directory that the test results will be available in once the container exits. -* `-f` gives the list of suites that should be executed -* Finally, list the specs to be run must be specified, space-separated. +* `-f` gives the list of suites to be run, space separated. Optional arguments are: * `-i` gives a name to a custom TNF container image. Supports local images, as well as images from external registries. -* `-k` gives a path to one or more kubeconfig files soto be used by the container to authenticate with the cluster. Paths must be separated by a colon. +* `-k` gives a path to one or more kubeconfig files to be used by the container to authenticate with the cluster. Paths must be separated by a colon. * `-n` gives the network mode of the container. Defaults to `bridge`. See the [docker run --network parameter reference](https://docs.docker.com/engine/reference/run/#network-settings) for more information on how to configure network settings. * `-s` gives the name of tests that should be skipped @@ -163,7 +177,7 @@ The autodiscovery first looks for paths in the `$KUBECONFIG` environment variabl ./run-tnf-container.sh -k ~/.kube/config -t ~/tnf/config -o ~/tnf/output -f diagnostic access-control -s access-control-host-resource-PRIVILEGED_POD ``` -*Note*: Tests must be specified after all other arguments! see [General tests](#general-tests) for a list of available keywords. +See [General tests](#general-tests) for a list of available keywords. *Note*: The `run-tnf-container.sh` script performs autodiscovery of selected TNF environment variables. Currently supported environment variables include: @@ -259,7 +273,6 @@ cd test-network-function ### Building the Tests In order to build the test executable, first make sure you have satisfied the [dependencies](#dependencies). - ```shell script make build-cnf-tests ``` @@ -292,7 +305,7 @@ cd test-network-function && ./test-network-function.test --help *Gotcha:* The generic test suite requires that the CNF has both `ping` and `ip` binaries installed. Please add them manually if the CNF under test does not include these. Automated installation of missing dependencies is targeted for a future version. - +*Gotcha:* check that OCP cluster has resources to deploy [debug image](#check-cluster-resources) ## Available Test Specs There are two categories for CNF tests; 'General' and 'CNF-specific' (TODO). From 0f914c9295f39ab0b43481e6e5d9be34a69304c1 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 27 Aug 2021 09:18:13 -0500 Subject: [PATCH 027/344] bugfix loglevel container (#325) --- script/run-container.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/run-container.sh b/script/run-container.sh index 2a521bf4a..c3f749be9 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -25,6 +25,7 @@ CONTAINER_DEFAULT_NETWORK_MODE=bridge CONTAINER_DEFAULT_TNF_MINIKUBE_ONLY=false CONTAINER_DEFAULT_TNF_NON_INTRUSIVE_ONLY=true CONTAINER_DEFAULT_TNF_DISABLE_CONFIG_AUTODISCOVER=false +LOG_LEVEL_DEFAULT=info get_container_tnf_kubeconfig_path_from_index() { local local_path_index="$1" @@ -79,7 +80,7 @@ CONTAINER_NETWORK_MODE="${CONTAINER_NETWORK_MODE:-$CONTAINER_DEFAULT_NETWORK_MOD CONTAINER_TNF_MINIKUBE_ONLY="${TNF_MINIKUBE_ONLY:-$CONTAINER_DEFAULT_TNF_MINIKUBE_ONLY}" CONTAINER_TNF_NON_INTRUSIVE_ONLY="${TNF_NON_INTRUSIVE_ONLY:-$CONTAINER_DEFAULT_TNF_NON_INTRUSIVE_ONLY}" CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER="${TNF_DISABLE_CONFIG_AUTODISCOVER:-$CONTAINER_DEFAULT_TNF_DISABLE_CONFIG_AUTODISCOVER}" - +LOG_LEVEL="${LOG_LEVEL:-$LOG_LEVEL_DEFAULT}" display_config_summary # Construct new $KUBECONFIG env variable containing all paths to kubeconfigs mounted to the container. @@ -106,7 +107,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_MINIKUBE_ONLY=$CONTAINER_TNF_MINIKUBE_ONLY \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ - -e LOG_LEVEL=$LOGLEVEL \ + -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ $TNF_CMD $OUTPUT_ARG $CONTAINER_TNF_DIR/claim $FOCUS_ARG $TNF_FOCUS_SUITES $SKIP_ARG $TNF_SKIP_SUITES "$@" From aaea790eebfcf46479df41d3ebf4679ed6707754 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 27 Aug 2021 17:20:31 -0400 Subject: [PATCH 028/344] Shamadi release 30 (#327) * remove unnecessary folder * release v3.0.0 --- fs-diff-test-utils/Dockerfile.podman-image | 3 -- fs-diff-test-utils/diff-fs.sh | 7 ---- fs-diff-test-utils/privilaged-pod.yaml | 49 ---------------------- version.json | 2 +- 4 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 fs-diff-test-utils/Dockerfile.podman-image delete mode 100755 fs-diff-test-utils/diff-fs.sh delete mode 100644 fs-diff-test-utils/privilaged-pod.yaml diff --git a/fs-diff-test-utils/Dockerfile.podman-image b/fs-diff-test-utils/Dockerfile.podman-image deleted file mode 100644 index 2f2e94e83..000000000 --- a/fs-diff-test-utils/Dockerfile.podman-image +++ /dev/null @@ -1,3 +0,0 @@ -FROM centos:8.3.2011 -RUN yum -y install podman -COPY diff-fs.sh /diff-fs.sh \ No newline at end of file diff --git a/fs-diff-test-utils/diff-fs.sh b/fs-diff-test-utils/diff-fs.sh deleted file mode 100755 index 705e859a5..000000000 --- a/fs-diff-test-utils/diff-fs.sh +++ /dev/null @@ -1,7 +0,0 @@ -result=`podman diff $1 | cut -d " " -f 2 | grep -E "(^/var/lib/rpm)|(^/var/lib/dpkg)|(^/bin)|(^/sbin)|(^/lib)|(^/lib64)|(^/usr/bin)|(^/usr/sbin)|(^/usr/lib)|(^/usr/lib64)"` -if [ -z "${result}" ] -then -echo empty -else -echo $result -fi \ No newline at end of file diff --git a/fs-diff-test-utils/privilaged-pod.yaml b/fs-diff-test-utils/privilaged-pod.yaml deleted file mode 100644 index b0cb7e11e..000000000 --- a/fs-diff-test-utils/privilaged-pod.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: node-master - name: node-master - namespace: tnf -spec: - # hostPID: true - containers: - - command: - - tail - - -f - - /dev/null - image: quay.io/isaacdorfman/centos-podman - name: master - resources: - limits: - memory: 512Mi - cpu: 0.25 - volumeMounts: - - mountPath: /var/lib/containers - name: var-lib-containers - - mountPath: /var/run/containers - name: var-run-containers - - mountPath: /run/runc - name: runc-dir - - mountPath: /var/run/containers/storage/overlay-containers - name: overlay-containers - securityContext: - privileged: true - restartPolicy: Always - volumes: - - name: var-lib-containers - hostPath: - path: /var/lib/containers - type: Directory - - name: var-run-containers - hostPath: - path: /var/run/containers - type: Directory - - name: runc-dir - hostPath: - path: /run/runc - type: Directory - - name: overlay-containers - hostPath: - path: /var/run/containers/storage/overlay-containers - type: Directory \ No newline at end of file diff --git a/version.json b/version.json index 170080018..cb475e476 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "tag": "v2.0.0" + "tag": "v3.0.0" } From 5bd77cc7b321b74c16e8e8df7359ee8807f7b00c Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 27 Aug 2021 17:40:13 -0400 Subject: [PATCH 029/344] fsdiff test fix (#326) remove the need to discover fsdiff pod update the fsdiff handler remove unnecessary script and folder from code base --- pkg/config/autodiscover/autodiscover.go | 1 + .../autodiscover/autodiscover_partner.go | 11 ------ pkg/config/autodiscover/pod_info.go | 1 + pkg/config/config.go | 2 - pkg/config/configsections/common.go | 2 - pkg/config/configsections/container.go | 1 + pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 38 +++++++++++-------- test-network-function/platform/suite.go | 35 +++++++++-------- 8 files changed, 43 insertions(+), 48 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index e4fb25ab6..ce35dddff 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -113,6 +113,7 @@ func buildContainersFromPodResource(pr *PodResource) (containers []configsection container.Namespace = pr.Metadata.Namespace container.PodName = pr.Metadata.Name container.ContainerName = containerResource.Name + container.NodeName = pr.Spec.NodeName container.DefaultNetworkDevice, err = pr.getDefaultNetworkDeviceFromAnnotations() if err != nil { log.Warnf("error encountered getting default network device: %s", err) diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index b8ac75297..f35da8d23 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -24,7 +24,6 @@ import ( const ( genericLabelName = "generic" orchestratorValue = "orchestrator" - fsDiffMasterValue = "fs_diff_master" ) // FindTestPartner completes a `configsections.TestPartner` from the current state of the cluster, @@ -38,14 +37,4 @@ func FindTestPartner(tp *configsections.TestPartner) { tp.ContainerConfigList = append(tp.ContainerConfigList, orchestrator) tp.TestOrchestratorID = orchestrator.ContainerIdentifier } - - if tp.FsDiffMasterContainerID.ContainerName == "" { - fsDiffMasterContainer, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: fsDiffMasterValue}) - if err == nil { - tp.ContainerConfigList = append(tp.ContainerConfigList, fsDiffMasterContainer) - tp.FsDiffMasterContainerID = fsDiffMasterContainer.ContainerIdentifier - } else { - log.Warnf("an error (%s) occurred when getting the FS Diff Master Container. Attempting to continue", err) - } - } } diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index c2645d873..fe0cc49c2 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -54,6 +54,7 @@ type PodResource struct { Containers []struct { Name string `json:"name"` } `json:"containers"` + NodeName string `json:"nodeName"` } `json:"spec"` Status struct { PodIPs []map[string]string `json:"podIPs"` diff --git a/pkg/config/config.go b/pkg/config/config.go index b6e770afc..c15011619 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -119,7 +119,6 @@ type TestEnvironment struct { // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} TestOrchestrator *Container - FsDiffMasterContainer *Container Config configsections.TestConfiguration // loaded tracks if the config has been loaded to prevent it being reloaded. loaded bool @@ -179,7 +178,6 @@ func (env *TestEnvironment) doAutodiscover() { env.PodsUnderTest = env.Config.PodsUnderTest env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] - env.FsDiffMasterContainer = env.PartnerContainers[env.Config.Partner.FsDiffMasterContainerID] log.Info(env.TestOrchestrator) log.Info(env.ContainersUnderTest) env.needsRefresh = false diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 62f420e80..c88c50bd2 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -60,8 +60,6 @@ type TestPartner struct { ContainerConfigList []ContainerConfig `yaml:"partnerContainers" json:"partnerContainers"` // TestOrchestratorID is the id of the partner container for conducting connectivity tests TestOrchestratorID ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` - // FsDiffMasterContainerID is the id of the partner container for conducting base image comparison - FsDiffMasterContainerID ContainerIdentifier `yaml:"fsDiffMasterContainer" json:"fsDiffMasterContainer"` } // TestTarget is a collection of resources under test diff --git a/pkg/config/configsections/container.go b/pkg/config/configsections/container.go index 97d03c829..7c7356e23 100644 --- a/pkg/config/configsections/container.go +++ b/pkg/config/configsections/container.go @@ -21,6 +21,7 @@ type ContainerIdentifier struct { Namespace string `yaml:"namespace" json:"namespace"` PodName string `yaml:"podName" json:"podName"` ContainerName string `yaml:"containerName" json:"containerName"` + NodeName string `yaml:"nodeName" json:"nodeName"` } // ContainerConfig contains the payload of container facets. diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 8fad03662..6cd2041ba 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -29,15 +29,15 @@ type CnfFsDiff struct { result int timeout time.Duration args []string - diff string } const ( - // SuccessfulOutputRegex matches a successfully run "fsdiff" command. That does not mean that no errors or drops - // occurred during the test. - SuccessfulOutputRegex = `(?m)empty\n` - // AcceptAllRegex matches all strings - AcceptAllRegex = `(?m)(.|\n)+` + bin = `(?m)\/bin` + sbin = `(?m)\/sbin` + lib = `(?m)\/lib` + err = `(?m)Error` + successfulOutputRegex = `(?m){}` + acceptAllRegex = `(?m)(.|\n)+` ) // Args returns the command line args for the test. @@ -71,10 +71,18 @@ func (p *CnfFsDiff) ReelFirst() *reel.Step { // ReelMatch checks if the test passed the first regex which means there were no installation on the container // or the second regex which accepts everything and means that something in the container was installed. func (p *CnfFsDiff) ReelMatch(pattern, before, match string) *reel.Step { - if pattern == SuccessfulOutputRegex { - p.result = tnf.SUCCESS - } else { + p.result = tnf.SUCCESS + switch pattern { + case lib: + p.result = tnf.FAILURE + case bin: p.result = tnf.FAILURE + case sbin: + p.result = tnf.FAILURE + case err: + p.result = tnf.ERROR + case successfulOutputRegex: + p.result = tnf.SUCCESS } return nil } @@ -89,20 +97,20 @@ func (p *CnfFsDiff) ReelEOF() { } // Command returns command line args for checking the fs difference between a container and it's image -func Command(containerID string) []string { - return []string{"/diff-fs.sh", containerID} +func Command(containerID, nodeName string) []string { + return []string{"echo", "-e", "\"chroot /host\n\"", "podman", "diff", "--format", "json", containerID, "|", "oc", "debug", "node/" + nodeName} } // NewFsDiff creates a new `FsDiff` test which checks the fs difference between a container and it's image -func NewFsDiff(timeout time.Duration, containerID string) *CnfFsDiff { +func NewFsDiff(timeout time.Duration, containerID, nodeName string) *CnfFsDiff { return &CnfFsDiff{ - result: tnf.ERROR, + result: tnf.SUCCESS, timeout: timeout, - args: Command(containerID), + args: Command(containerID, nodeName), } } // GetReelFirstRegularExpressions returns the regular expressions used for matching in ReelFirst. func (p *CnfFsDiff) GetReelFirstRegularExpressions() []string { - return []string{SuccessfulOutputRegex, AcceptAllRegex} + return []string{err, bin, sbin, lib, successfulOutputRegex, acceptAllRegex} } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 3cf96d3b1..db49a3c54 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -32,7 +32,6 @@ import ( "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/cnffsdiff" @@ -116,23 +115,25 @@ func testContainerIsRedHatRelease(cut *config.Container) { func testContainersFsDiff(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredBaseImageIdentifier) ginkgo.It(testID, func() { - fsDiffContainer := env.FsDiffMasterContainer - if fsDiffContainer != nil { - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - containerName := cut.Oc.GetPodContainerName() - context := cut.Oc - ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) - testContainerFsDiff(fsDiffContainer.Oc, context) + var badContainers []string + for _, cut := range env.ContainersUnderTest { + podName := cut.Oc.GetPodName() + containerName := cut.Oc.GetPodContainerName() + context := cut.Oc + nodeName := cut.ContainerConfiguration.NodeName + ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) + testResult, err := testContainerFsDiff(nodeName, context) + if testResult != tnf.SUCCESS || err != nil { + badContainers = append(badContainers, containerName) + ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) } - } else { - log.Warn("no fs diff container is configured, cannot run fs diff test") } + gomega.Expect(badContainers).To(gomega.BeNil()) }) } // testContainerFsDiff test that the CUT didn't install new packages after starting, and report through Ginkgo. -func testContainerFsDiff(masterPodOc, targetContainerOC *interactive.Oc) { +func testContainerFsDiff(nodeName string, targetContainerOC *interactive.Oc) (int, error) { defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) targetContainerOC.GetExpecter() containerIDTester := containerid.NewContainerID(common.DefaultTimeout) @@ -142,13 +143,11 @@ func testContainerFsDiff(masterPodOc, targetContainerOC *interactive.Oc) { gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) containerID := containerIDTester.GetID() - - fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID) - test, err = tnf.NewTest(masterPodOc.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, masterPodOc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err = test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + context := common.GetContext() + fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID, nodeName) + test, err = tnf.NewTest(context.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) + return test.Run() } func getMcKernelArguments(context *interactive.Context, mcName string) map[string]string { From af9cbee720474bd4e818e3ae9d4891fcaa9d05a5 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Mon, 30 Aug 2021 18:17:51 -0400 Subject: [PATCH 030/344] Add operator to the container run in CI workflow (#330) Did this change together with Salah. --- .github/workflows/pre-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 0d880233d..75ed1ac55 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -147,7 +147,7 @@ jobs: run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator # Push the new unstable TNF image to Quay.io. From 7296b936d51ae5ca290006cd445e14c430e221ab Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 31 Aug 2021 15:21:37 -0400 Subject: [PATCH 031/344] Update README.md (#329) --- README.md | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 12b101a3d..2da69058a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ In the diagram above: The Test Network Function support auto-configuration using labels and annotations, but also a static configuration using a file. The following sections describe how to configure the TNF via labels/annotation and the corresponding settings in the config file. A sample config file can be found [here](test-network-function/tnf_config.yml). ### targetPodLabels -The goal of this section is to specify the label to be used to identify the cnf under test pods. So for example, with the default configuration: +The goal of this section is to specify the label to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The namespace field is just a prefix for the label name, e.g. with the default configuration: ```shell script targetPodLabels: - namespace: test-network-function.com @@ -33,19 +33,18 @@ targetPodLabels: value: target ``` -The corresponding label prefix is: +The corresponding label used to match pods is: ```shell script test-network-function.com/generic: target ``` -When labelling a pod to be discovered and tested, the discovered pods are **in addition to** the ones -explicitly configured in the testTarget sections of the config file. +Once the pods are found, all of their containers are also added to the target container list. ### testTarget #### podsUnderTest / containersUnderTest -This section is usually not required if labels defined in the section above cover all resources that should be tested. It's highly recommended that the labels shoud be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. +This section is usually not required if labels defined in the section above cover all resources that should be tested. If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, instrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. The pods and containers explicitly configured here are added to the target pod/container lists populated through label matching. -The autodiscovery mechanism will also attempt to identify the default network device and all the IP addresses of the pods it +For both configured and discovered pods/containers, the autodiscovery mechanism will attempt to identify the default network device and all the IP addresses of the pods it needs for network connectivity tests, though that information can be explicitly set using annotations if needed. For Pod IPs: * The annotation `test-network-function.com/multusips` is the highest priority, and must contain a JSON-encoded list of @@ -63,15 +62,16 @@ be seen in [cnf-certification-test-partner](https://github.com/test-network-func the first entry found with `"default"=true` is used. This annotation is automatically managed in OpenShift but may not be present in K8s. +If multus IP addresses are dicovered or configured, the partner pod needs to be deployed in the same namespace as the multus network interface for the connectivity test to pass. Refer to instruction [here](#specify-the-target-namespace-for-partner-pod-deployment). + If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. `ping`), it should be given the label `test-network-function.com/skip_connectivity_tests` to exclude it from those tests. The label value is not important, only its presence. Equivalent to `excludeContainersFromConnectivityTests` in the config file. -If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, instrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. #### operators -The section can be configured as well as auto discovered. For manual configuration, see the commented part of the [sample config](test-network-function/tnf_config.yml). For auto discovery: +The section can be configured as well as auto discovered. For manual configuration, see the commented part of the [sample config](test-network-function/tnf_config.yml). For autodiscovery: * CSVs to be tested by the `operator` spec are identified with the `test-network-function-com/operator=target` label. Any value is permitted but `target` is used here for consistency with the other specs. @@ -89,7 +89,7 @@ This section can also be discovered automatically and should be left commented o The `certifiedcontainerinfo` and `certifiedoperatorinfo` sections contain information about CNFs and Operators that are to be checked for certification status on Red Hat catalogs. -## Runtime environement variables to skip or include tests +## Runtime environement variables ### Turn off openshift required tests When test on CNFs that run on k8s only environment, execute shell command below before compile tool and run test shell script. @@ -111,14 +111,24 @@ export TNF_NON_INTRUSIVE_ONLY=false ``` ### Specifiy the location of the partner repo -This env var is mandatory. -You must clone the partner [repo](https://github.com/test-network-function/cnf-certification-test-partner) -and set TNF_PARTNER_SRC_DIR to point to it. +This env var is optional, but highly recommended if running the test suite from a clone of this github repo. It's not needed or used if running the tnf image. + +To set it, clone the partner [repo](https://github.com/test-network-function/cnf-certification-test-partner) and set TNF_PARTNER_SRC_DIR to point to it. ```shell script -export TNF_PARTNER_SRC_DIR=~/code/cnf-certification-test-partner +export TNF_PARTNER_SRC_DIR=/home/userid/code/cnf-certification-test-partner ``` +When this variable is set, the run-cnf-suites.sh script will deploy/refresh the partner deployments/pods in the cluster before starting the test run. + +### Specify the target namespace for partner pod deployment +Set this variable to deploy partner pods in a custom namespace instead of the default `tnf` namesapce. + +```shell-script +export NAMESPACE_TO_GENERATE="CNF-ns" +``` + + ### Execute test suites from openshift-kni/cnf-feature-deploy The test suites from openshift-kni/cnf-feature-deploy can be run prior to the actual CNF certification test execution and the results are incorporated in the same claim file if the following environment variable is set: @@ -291,7 +301,7 @@ Run any combination of the suites keywords listed at in the [General tests](#gen ./run-cnf-suites.sh -f diagnostic lifecycle ./run-cnf-suites.sh -f diagnostic networking operator ./run-cnf-suites.sh -f diagnostic platform-alteration -./run-cnf-suites.sh -f diagnostic generic lifecycle affiliated-certification operator +./run-cnf-suites.sh -f diagnostic lifecycle affiliated-certification operator ``` By default the claim file will be output into the same location as the test executable. The `-o` argument for From 30ef1e399c837c648984d241e1cc0814b50fee40 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 31 Aug 2021 15:41:31 -0400 Subject: [PATCH 032/344] Remove the duplicate OVP query test and clean up the operator suite (#332) --- pkg/tnf/testcases/base.go | 15 +++ test-network-function/operator/suite.go | 120 ++++++++---------------- 2 files changed, 55 insertions(+), 80 deletions(-) diff --git a/pkg/tnf/testcases/base.go b/pkg/tnf/testcases/base.go index 693c6c95d..572fab679 100644 --- a/pkg/tnf/testcases/base.go +++ b/pkg/tnf/testcases/base.go @@ -361,3 +361,18 @@ func GetConfiguredPodTests() (cnfTests []string) { log.WithField("cnfTests", cnfTests).Infof("got all tests from %s.", ConfiguredTestFile) return cnfTests } + +// GetConfiguredOperatorTests loads the `configuredTestFile` and extracts +// the names of test groups from it. +func GetConfiguredOperatorTests() (operatorTests []string) { + configuredTests, err := LoadConfiguredTestFile(ConfiguredTestFile) + if err != nil { + log.Errorf("failed to load %s, continuing with no tests", ConfiguredTestFile) + return []string{} + } + for _, configuredTest := range configuredTests.OperatorTest { + operatorTests = append(operatorTests, configuredTest.Name) + } + log.WithField("operatorTests", operatorTests).Infof("got all tests from %s.", ConfiguredTestFile) + return operatorTests +} diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index d3bd4be8c..9d574899a 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -20,7 +20,6 @@ import ( "fmt" "path" "strings" - "time" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" @@ -32,7 +31,6 @@ import ( ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/internal/api" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -45,19 +43,12 @@ import ( const ( configuredTestFile = "testconfigure.yml" // The default test timeout. - defaultTimeoutSeconds = 10 - // timeout for eventually call - eventuallyTimeoutSeconds = 30 - // interval of time - interval = 1 - testSpecName = "operator" - subscriptionTest = "SUBSCRIPTION_INSTALLED" + testSpecName = "operator" ) var ( - defaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second - context *interactive.Context - err error + context *interactive.Context + err error // checkSubscriptionTestPath is the file location of the uncordon.json test case relative to the project root. checkSubscriptionTestPath = path.Join("pkg", "tnf", "handlers", "checksubscription", "check-subscription.json") @@ -84,9 +75,7 @@ var _ = ginkgo.Describe(testSpecName, func() { defer ginkgo.GinkgoRecover() ginkgo.When("a local shell is spawned", func() { - goExpectSpawner := interactive.NewGoExpectSpawner() - var spawner interactive.Spawner = goExpectSpawner - context, err = interactive.SpawnShell(&spawner, defaultTimeout, interactive.Verbose(true)) + context = common.GetContext() ginkgo.It("should be created without error", func() { gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(context).ToNot(gomega.BeNil()) @@ -143,79 +132,50 @@ func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsection } func itRunsTestsOnOperator() { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsCertifiedIdentifier) - ginkgo.It(testID, func() { - operatorsToQuery, operatorsInTest := getConfig() - if len(operatorsToQuery) > 0 { - certAPIClient := api.NewHTTPClient() - for _, certified := range operatorsToQuery { - // Care: this test takes some time to run, failures at later points while before this has finished may be reported as a failure here. Read the failure reason carefully. - ginkgo.By(fmt.Sprintf("should eventually be verified as certified (operator %s/%s)", certified.Organization, certified.Name)) - defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) - certified := certified // pin - gomega.Eventually(func() bool { - isCertified := certAPIClient.IsOperatorCertified(certified.Organization, certified.Name) - return isCertified - }, eventuallyTimeoutSeconds, interval).Should(gomega.BeTrue()) - } - } - if common.IsMinikube() { - log.Info("Minikube detected: skipping operators test.") - return - } - if len(operatorsInTest) == 0 { - ginkgo.Fail("No operators to test were discovered.") - } + if common.IsMinikube() { + log.Info("Minikube detected: skipping operators test.") + return + } - for _, op := range operatorsInTest { - // TODO: Gather facts for operator - for _, testType := range op.Tests { - testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) - gomega.Expect(testFile).ToNot(gomega.BeNil()) - gomega.Expect(err).To(gomega.BeNil()) - testConfigure := testcases.ContainsConfiguredTest(testFile.OperatorTest, testType) - renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Operator, testType) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) - for _, testCase := range renderedTestCase.TestCase { - if testCase.SkipTest { - continue - } - if testCase.ExpectedType == testcases.Function { - for _, val := range testCase.ExpectedStatus { - testCase.ExpectedStatusFn(op.Name, testcases.StatusFunctionType(val)) - } - } - name := agrName(op.Name, op.SubscriptionName, testCase.Name) - args := []interface{}{name, op.Namespace} - runTestsOnOperator(args, name, op.Namespace, testCase) - } + for _, testType := range testcases.GetConfiguredOperatorTests() { + testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) + gomega.Expect(testFile).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + testConfigure := testcases.ContainsConfiguredTest(testFile.OperatorTest, testType) + renderedTestCase, err := testConfigure.RenderTestCaseSpec(testcases.Operator, testType) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(renderedTestCase).ToNot(gomega.BeNil()) + for _, testCase := range renderedTestCase.TestCase { + if testCase.SkipTest { + continue } + runTestsOnOperator(testCase) } - }) -} - -func agrName(operatorName, subName, testName string) string { - name := operatorName - if testName == subscriptionTest { - name = subName } - return name } //nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. -func runTestsOnOperator(args []interface{}, name, namespace string, testCmd testcases.BaseTestCase) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorInstallStatusIdentifier) + "-" + testCmd.Name +func runTestsOnOperator(testCase testcases.BaseTestCase) { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorInstallStatusIdentifier) + "-" + testCase.Name ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestOperatorInstallStatusIdentifier) - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") - opInTest := operator.NewOperator(cmdArgs, name, namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, defaultTimeout) - gomega.Expect(opInTest).ToNot(gomega.BeNil()) - test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + for _, op := range config.GetTestEnvironment().Config.Operators { + if testCase.ExpectedType == testcases.Function { + for _, val := range testCase.ExpectedStatus { + testCase.ExpectedStatusFn(op.Name, testcases.StatusFunctionType(val)) + } + } + name := op.Name + args := []interface{}{name, op.Namespace} + cmdArgs := strings.Split(fmt.Sprintf(testCase.Command, args...), " ") + opInTest := operator.NewOperator(cmdArgs, name, op.Namespace, testCase.ExpectedStatus, testCase.ResultType, testCase.Action, common.DefaultTimeout) + gomega.Expect(opInTest).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + } }) } From c9e34eff61586ac7a8e727737850145529b1c5d7 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Sep 2021 09:51:34 -0400 Subject: [PATCH 033/344] Update the partner namespace variable and fix for the container case (#333) --- .github/workflows/pre-main.yaml | 2 +- README.md | 2 +- script/run-container.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 75ed1ac55..8d1fafca8 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -19,7 +19,7 @@ env: TNF_OUTPUT_DIR: /tmp/tnf/output TNF_SRC_URL: 'https://github.com/${{ github.repository }}' TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${IMAGE_NAME}:${IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}' - NAMESPACE_TO_GENERATE: tnf + TNF_PARTNER_NAMESPACE: tnf TNF_PARTNER_DIR: '/usr/tnf-partner' TNF_PARTNER_SRC_DIR: '${TNF_PARTNER_DIR}/src' diff --git a/README.md b/README.md index 2da69058a..fe7687372 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ When this variable is set, the run-cnf-suites.sh script will deploy/refresh the Set this variable to deploy partner pods in a custom namespace instead of the default `tnf` namesapce. ```shell-script -export NAMESPACE_TO_GENERATE="CNF-ns" +export TNF_PARTNER_NAMESPACE="CNF-ns" ``` diff --git a/script/run-container.sh b/script/run-container.sh index c3f749be9..c50fc6026 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -107,6 +107,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_MINIKUBE_ONLY=$CONTAINER_TNF_MINIKUBE_ONLY \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ + -e TNF_PARTNER_NAMESPACE=$TNF_PARTNER_NAMESPACE \ -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ From a9baed84577c037dcfcd8bcb914b5dfc3634f61f Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:29:03 +0300 Subject: [PATCH 034/344] Change the default network mode to host in run-tnf-container.sh (#331) * Change the default network mode to host in run-tnf-container.sh * update the readme * update the readme conflict * update the = * update the export to the top * update the export to the top * update the -n redme * update the -n readme * add the $ in network mode Co-authored-by: Salaheddine Hamadi --- README.md | 3 ++- run-tnf-container.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fe7687372..d8f2f91f7 100644 --- a/README.md +++ b/README.md @@ -177,9 +177,10 @@ Optional arguments are: * `-i` gives a name to a custom TNF container image. Supports local images, as well as images from external registries. * `-k` gives a path to one or more kubeconfig files to be used by the container to authenticate with the cluster. Paths must be separated by a colon. -* `-n` gives the network mode of the container. Defaults to `bridge`. See the [docker run --network parameter reference](https://docs.docker.com/engine/reference/run/#network-settings) for more information on how to configure network settings. +* `-n` gives the network mode of the container. Defaults set to `host`, which requires selinux to be disabled. Alternatively, `bridge` mode can be used with selinux if TNF_CONTAINER_CLIENT is set to `docker` or running the test as root. See the [docker run --network parameter reference](https://docs.docker.com/engine/reference/run/#network-settings) for more information on how to configure network settings. * `-s` gives the name of tests that should be skipped + If `-k` is not specified, autodiscovery is performed. The autodiscovery first looks for paths in the `$KUBECONFIG` environment variable on the host system, and if the variable is not set or is empty, the default configuration stored in `$HOME/.kube/config` is checked. diff --git a/run-tnf-container.sh b/run-tnf-container.sh index 3120bf5e5..4e4f133e6 100755 --- a/run-tnf-container.sh +++ b/run-tnf-container.sh @@ -29,6 +29,7 @@ export TNF_CMD="./run-cnf-suites.sh" export OUTPUT_ARG="-o" export FOCUS_ARG="-f" export SKIP_ARG="-s" +export CONTAINER_NETWORK_MODE='host' usage() { read -d '' usage_prompt <<- EOF @@ -178,7 +179,7 @@ while [[ $1 == -* ]]; do echo "-n requires an argument" 1>&2 exit 1 fi - echo "-n CONTAINER_NETWORK_MODE" + echo "-n $CONTAINER_NETWORK_MODE" ;; -d) if (($# > 1)); then export DNS_ARG=$2; shift 2 From ec7930bd6e439e16ff1b324b69fbce183de23a1d Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Sep 2021 13:05:10 -0400 Subject: [PATCH 035/344] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8f2f91f7..9f65a35fa 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ not important, only its presence. Equivalent to `excludeContainersFromConnectivi The section can be configured as well as auto discovered. For manual configuration, see the commented part of the [sample config](test-network-function/tnf_config.yml). For autodiscovery: -* CSVs to be tested by the `operator` spec are identified with the `test-network-function-com/operator=target` +* CSVs to be tested by the `operator` spec are identified with the `test-network-function.com/operator=target` label. Any value is permitted but `target` is used here for consistency with the other specs. * Defining which tests are to be run on the operator is done using the `test-network-function.com/operator_tests` annotation. This is equivalent to the `test-network-function.com/container_tests` and behaves the same. From 9cf5d44142b1783a31bc6abf9dfd418d41bd1e28 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Sep 2021 13:26:51 -0400 Subject: [PATCH 036/344] fix the test identifier for the redhat release check (#335) --- test-network-function/platform/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index db49a3c54..649694fef 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -87,7 +87,7 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { // testIsRedHatRelease fetch the configuration and test containers attached to oc is Red Hat based. func testIsRedHatRelease(env *config.TestEnvironment) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestIsRedHatReleaseIdentifier) ginkgo.It(testID, func() { ginkgo.By("should report a proper Red Hat version") defer results.RecordResult(identifiers.TestIsRedHatReleaseIdentifier) From e62d62182b8397fb70d74b15cb9fae34b70a6390 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Sep 2021 13:53:21 -0400 Subject: [PATCH 037/344] Update run-container.sh (#336) --- script/run-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-container.sh b/script/run-container.sh index c50fc6026..31c294407 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -72,7 +72,7 @@ for local_path_index in "${!local_kubeconfig_paths[@]}"; do container_path=$(get_container_tnf_kubeconfig_path_from_index $local_path_index) container_tnf_kubeconfig_paths+=($container_path) - container_tnf_kubeconfig_volume_bindings+=("$local_path:$container_path:ro") + container_tnf_kubeconfig_volume_bindings+=("$local_path:$container_path:Z") done TNF_IMAGE="${TNF_IMAGE:-$TNF_OFFICIAL_IMAGE}" From 5cb349f08d17430b0d1b564a25750698f48d9d82 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Sep 2021 14:50:09 -0400 Subject: [PATCH 038/344] Fix for the exclude from network test label handling (#338) --- pkg/config/config.go | 2 ++ test-network-function/networking/suite.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index c15011619..48c6881cf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -171,6 +171,8 @@ func (env *TestEnvironment) doAutodiscover() { log.Infof("Test Configuration: %+v", *env) + env.ContainersToExcludeFromConnectivityTests = make(map[configsections.ContainerIdentifier]interface{}) + for _, cid := range env.Config.ExcludeContainersFromConnectivityTests { env.ContainersToExcludeFromConnectivityTests[cid] = "" } diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index a92bd513b..b3da11e3c 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -98,6 +98,9 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestICMPv4ConnectivityIdentifier) ginkgo.It(testID, func() { for _, cut := range env.ContainersUnderTest { + if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { + continue + } for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), From 7dfabcdb1fd4eda7867a4cb4bbded8c24f96c1cd Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 2 Sep 2021 13:34:39 -0400 Subject: [PATCH 039/344] Add oc debug image id support (#339) * Add oc debug image id support * Update run-container script with the new env var --- .gitignore | 2 +- README.md | 9 +++++++-- .../bootconfigentries/bootconfigentries.go | 3 ++- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 3 ++- pkg/tnf/handlers/common/command.go | 16 ++++++++++++++++ pkg/tnf/handlers/common/doc.go | 18 ++++++++++++++++++ pkg/tnf/handlers/nodedebug/nodedebug.go | 3 ++- .../handlers/nodehugepages/nodehugepages.go | 3 ++- pkg/tnf/handlers/nodetainted/nodetainted.go | 3 ++- .../handlers/readbootconfig/readbootconfig.go | 5 +++-- .../handlers/readremotefile/readremotefile.go | 3 ++- .../sysctlallconfigsargs.go | 3 ++- script/run-container.sh | 1 + test-network-function/common/env.go | 5 +++++ test-network-function/suite_test.go | 2 ++ 15 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 pkg/tnf/handlers/common/command.go create mode 100644 pkg/tnf/handlers/common/doc.go diff --git a/.gitignore b/.gitignore index b8574954e..b22df2f98 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,4 @@ pkg/tnf/reel/mocks/mock_reel.go # NOTE: mock_expect.go is tracked, as it is generated from source externally. pkg/tnf/interactive/mocks/mock_spawner.go temp/ -tnf +/tnf \ No newline at end of file diff --git a/README.md b/README.md index 9f65a35fa..a4bddbeb9 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,6 @@ The section can be configured as well as auto discovered. For manual configurati * CSVs to be tested by the `operator` spec are identified with the `test-network-function.com/operator=target` label. Any value is permitted but `target` is used here for consistency with the other specs. -* Defining which tests are to be run on the operator is done using the `test-network-function.com/operator_tests` -annotation. This is equivalent to the `test-network-function.com/container_tests` and behaves the same. * `test-network-function.com/subscription_name` is optional and should contain a JSON-encoded string that's the name of the subscription for this CSV. If unset, the CSV name will be used. @@ -128,6 +126,13 @@ Set this variable to deploy partner pods in a custom namespace instead of the de export TNF_PARTNER_NAMESPACE="CNF-ns" ``` +### Specify the image id to be used with oc debug commands +In disconnected environment, only specific versions of images are mirrored to the local repo. For the `oc debug` command (used by a number of tests) to work, set TNF_OC_DEBUG_IMAGE_ID: + +```shell-script +export TNF_OC_DEBUG_IMAGE_ID="quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0f5ce898fbad3671fecd6f797130f950fb1abdbaf0d8154c22e2b59c74e3a918" +``` + ### Execute test suites from openshift-kni/cnf-feature-deploy The test suites from openshift-kni/cnf-feature-deploy can be run prior to the actual CNF certification test execution and the results are incorporated in the same claim file if the following environment variable is set: diff --git a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go index 04f18925c..bdcce74c2 100644 --- a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go +++ b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go @@ -21,6 +21,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -43,7 +44,7 @@ func NewBootConfigEntries(timeout time.Duration, nodeName string) *BootConfigEnt timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"ls /host/boot/loader/entries/\"", "|", "oc", "debug", "--preserve-pod=true", "node/" + nodeName, + "echo", "\"ls /host/boot/loader/entries/\"", "|", common.GetOcDebugCommand(), "--preserve-pod=true", "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 6cd2041ba..1efcc65e7 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -20,6 +20,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -98,7 +99,7 @@ func (p *CnfFsDiff) ReelEOF() { // Command returns command line args for checking the fs difference between a container and it's image func Command(containerID, nodeName string) []string { - return []string{"echo", "-e", "\"chroot /host\n\"", "podman", "diff", "--format", "json", containerID, "|", "oc", "debug", "node/" + nodeName} + return []string{"echo", "-e", "\"chroot /host\n\"", "podman", "diff", "--format", "json", containerID, "|", common.GetOcDebugCommand(), "node/" + nodeName} } // NewFsDiff creates a new `FsDiff` test which checks the fs difference between a container and it's image diff --git a/pkg/tnf/handlers/common/command.go b/pkg/tnf/handlers/common/command.go new file mode 100644 index 000000000..ed84c8896 --- /dev/null +++ b/pkg/tnf/handlers/common/command.go @@ -0,0 +1,16 @@ +package common + +// OcDebugImageID can be set by the test application that uses the test handlers which require the oc debug command to work with a specific image version +var OcDebugImageID string + +const ( + ocDebugCmdBase = "oc debug" +) + +// GetOcDebugCommand returns the command base for any test handler that uses the oc debug command +func GetOcDebugCommand() string { + if len(OcDebugImageID) > 0 { + return ocDebugCmdBase + " --image " + OcDebugImageID + } + return ocDebugCmdBase +} diff --git a/pkg/tnf/handlers/common/doc.go b/pkg/tnf/handlers/common/doc.go new file mode 100644 index 000000000..473281eab --- /dev/null +++ b/pkg/tnf/handlers/common/doc.go @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package common provides utilities shared by handlers +package common diff --git a/pkg/tnf/handlers/nodedebug/nodedebug.go b/pkg/tnf/handlers/nodedebug/nodedebug.go index b79f68fbe..0d37e86a1 100644 --- a/pkg/tnf/handlers/nodedebug/nodedebug.go +++ b/pkg/tnf/handlers/nodedebug/nodedebug.go @@ -21,6 +21,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -50,7 +51,7 @@ func NewNodeDebug(timeout time.Duration, nodeName, command string, trim, split b timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "-e", "\"chroot /host\n\"", command, "|", "oc", "debug", "node/" + nodeName, + "echo", "-e", "\"chroot /host\n\"", command, "|", common.GetOcDebugCommand(), "node/" + nodeName, }, Trim: trim, Split: split, diff --git a/pkg/tnf/handlers/nodehugepages/nodehugepages.go b/pkg/tnf/handlers/nodehugepages/nodehugepages.go index edfeb3e49..3375a4b80 100644 --- a/pkg/tnf/handlers/nodehugepages/nodehugepages.go +++ b/pkg/tnf/handlers/nodehugepages/nodehugepages.go @@ -22,6 +22,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -47,7 +48,7 @@ func NewNodeHugepages(timeout time.Duration, node string, hugepagesz, hugepages timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"grep -E 'HugePages_Total:|Hugepagesize:' /proc/meminfo\"", "|", "oc", "debug", "node/" + node, + "echo", "\"grep -E 'HugePages_Total:|Hugepagesize:' /proc/meminfo\"", "|", common.GetOcDebugCommand(), "node/" + node, }, } } diff --git a/pkg/tnf/handlers/nodetainted/nodetainted.go b/pkg/tnf/handlers/nodetainted/nodetainted.go index 543398b74..1151c429a 100644 --- a/pkg/tnf/handlers/nodetainted/nodetainted.go +++ b/pkg/tnf/handlers/nodetainted/nodetainted.go @@ -20,6 +20,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -41,7 +42,7 @@ func NewNodeTainted(timeout time.Duration, nodeName string) *NodeTainted { timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "cat", "/proc/sys/kernel/tainted", "|", "oc", "debug", "node/" + nodeName, + "echo", "cat", "/proc/sys/kernel/tainted", "|", common.GetOcDebugCommand(), "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/readbootconfig/readbootconfig.go b/pkg/tnf/handlers/readbootconfig/readbootconfig.go index e8136e5d8..aae83fa75 100644 --- a/pkg/tnf/handlers/readbootconfig/readbootconfig.go +++ b/pkg/tnf/handlers/readbootconfig/readbootconfig.go @@ -20,6 +20,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -42,8 +43,8 @@ func NewReadBootConfig(timeout time.Duration, nodeName /*, entryName*/ string) * timeout: timeout, result: tnf.ERROR, args: []string{ - // "echo", "\"cat /host/boot/loader/entries/" + entryName + "\"", "|", "oc", "debug", "--preserve-pod=true", "node/" + nodeName, - "echo", "\"cat /host/boot/loader/entries/\\`ls /host/boot/loader/entries/ | sort | tail -n 1\\`\"", "|", "oc", "debug", "-q", "node/" + nodeName, + // "echo", "\"cat /host/boot/loader/entries/" + entryName + "\"", "|", common.GetOcDebugCommand(), "--preserve-pod=true", "node/" + nodeName, + "echo", "\"cat /host/boot/loader/entries/\\`ls /host/boot/loader/entries/ | sort | tail -n 1\\`\"", "|", common.GetOcDebugCommand(), "-q", "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/readremotefile/readremotefile.go b/pkg/tnf/handlers/readremotefile/readremotefile.go index 6d9d61e2d..3e4016c4e 100644 --- a/pkg/tnf/handlers/readremotefile/readremotefile.go +++ b/pkg/tnf/handlers/readremotefile/readremotefile.go @@ -20,6 +20,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -42,7 +43,7 @@ func NewReadRemoteFile(timeout time.Duration, nodeName, filePath string) *ReadRe timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"cat /host" + filePath + "\"", "|", "oc", "debug", "-q", "node/" + nodeName, + "echo", "\"cat /host" + filePath + "\"", "|", common.GetOcDebugCommand(), "-q", "node/" + nodeName, }, } } diff --git a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go index bbe11d991..419df701c 100644 --- a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go +++ b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go @@ -20,6 +20,7 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -43,7 +44,7 @@ func NewSysctlAllConfigsArgs(timeout time.Duration, nodeName string) *SysctlAllC timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"sysctl --system\"", "|", "oc", "debug", "-q", "node/" + nodeName, + "echo", "\"sysctl --system\"", "|", common.GetOcDebugCommand(), "-q", "node/" + nodeName, }, } } diff --git a/script/run-container.sh b/script/run-container.sh index 31c294407..38132f0af 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -108,6 +108,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ -e TNF_PARTNER_NAMESPACE=$TNF_PARTNER_NAMESPACE \ + -e TNF_OC_DEBUG_IMAGE_ID=$TNF_OC_DEBUG_IMAGE_ID \ -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index bd836e89d..d9fbc220d 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -86,6 +86,11 @@ func Intrusive() bool { return !b } +// GetOcDebugImageID is for running oc debug commands in a disconnected environment with a specific oc debug pod image mirrored +func GetOcDebugImageID() string { + return os.Getenv("TNF_OC_DEBUG_IMAGE_ID") +} + // logLevel retrieves the LOG_LEVEL environement vaiable func logLevel() string { return os.Getenv("LOG_LEVEL") diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index b45197ac4..738e52c84 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -34,6 +34,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/junit" "github.com/test-network-function/test-network-function/pkg/tnf" + tnfcommon "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" _ "github.com/test-network-function/test-network-function/test-network-function/accesscontrol" _ "github.com/test-network-function/test-network-function/test-network-function/certification" @@ -115,6 +116,7 @@ func TestTest(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) log.Info("Version: ", programVersion, " ( ", GitCommit, " )") common.SetLogLevel() + tnfcommon.OcDebugImageID = common.GetOcDebugImageID() // Initialize the claim with the start time, tnf version, etc. claimRoot := createClaimRoot() From 44c4be6330cb0a68e29bcb8ad9de2fd4cb65ad2c Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Thu, 2 Sep 2021 22:26:12 +0300 Subject: [PATCH 040/344] update_operator (#328) * update_operator * Update description TestOperatorInstallStatusIdentifier * Update CATALOG.md * Update identifiers.go Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- CATALOG.md | 4 ++-- test-network-function/identifiers/identifiers.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index c22f52c4b..1aec18284 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -179,7 +179,7 @@ Suggested Remediation|Ensure that your Operator is installed via OLM. Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. TODO: Describe operator scc check. +Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The Operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. Result Type|normative Suggested Remediation|Ensure that your Operator abides by the Operator Best Practices mentioned in the description. ### http://test-network-function.com/testcases/platform-alteration/base-image @@ -499,7 +499,7 @@ Runtime Binaries Required|`oc`, `cat`, `echo` Property|Description ---|--- Version|v1.0.0 -Description|An operator-specific test used to exercise the behavior of a given operator. In the current offering, we check if the operator ClusterServiceVersion (CSV) is installed properly. A CSV is a YAML manifest created from Operator metadata that assists the Operator Lifecycle Manager (OLM) in running the Operator. +Description|An operator-specific test used to exercise the behavior of a given operator. In the current offering, we check if the operator ClusterServiceVersion (CSV) is installed properly. A CSV is a YAML manifest created from Operator metadata that assists the Operator Lifecycle Manager (OLM) in running the Operator. Result Type|normative Intrusive|false Modifications Persist After Test|false diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 8a307368c..bdbdef48b 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -323,7 +323,8 @@ the same hacks.'`), Description: formDescription(TestOperatorInstallStatusIdentifier, `Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. -2. TODO: Describe operator scc check.`), +2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present +with no resourceNames under its rules.`), }, TestOperatorIsCertifiedIdentifier: { From f2fbbbd23ffc5323f8ea8e19924b216c3920c3c6 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 2 Sep 2021 16:08:43 -0400 Subject: [PATCH 041/344] Update folders to check in fsdiff (#334) * Update folders to check in fsdiff * address david's comment --- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 1efcc65e7..9a7dd0b21 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -33,9 +33,14 @@ type CnfFsDiff struct { } const ( - bin = `(?m)\/bin` - sbin = `(?m)\/sbin` - lib = `(?m)\/lib` + varlibrpm = `(?m)[\t|\s]\/var\/lib\/rpm[.]*` + varlibdpkg = `(?m)[\t|\s]\/var\/lib\/dpkg[.]*` + bin = `(?m)[\t|\s]\/bin[.]*` + sbin = `(?m)[\t|\s]\/sbin[.]*` + lib = `(?m)[\t|\s]\/lib[.]*` + usrbin = `(?m)[\t|\s]\/usr\/bin[.]*` + usrsbin = `(?m)[\t|\s]\/usr\/sbin[.]*` + usrlib = `(?m)[\t|\s]\/usr\/lib[.]*` err = `(?m)Error` successfulOutputRegex = `(?m){}` acceptAllRegex = `(?m)(.|\n)+` @@ -74,14 +79,10 @@ func (p *CnfFsDiff) ReelFirst() *reel.Step { func (p *CnfFsDiff) ReelMatch(pattern, before, match string) *reel.Step { p.result = tnf.SUCCESS switch pattern { - case lib: - p.result = tnf.FAILURE - case bin: - p.result = tnf.FAILURE - case sbin: - p.result = tnf.FAILURE case err: p.result = tnf.ERROR + case varlibrpm, varlibdpkg, bin, sbin, lib, usrbin, usrsbin, usrlib: + p.result = tnf.FAILURE case successfulOutputRegex: p.result = tnf.SUCCESS } @@ -113,5 +114,5 @@ func NewFsDiff(timeout time.Duration, containerID, nodeName string) *CnfFsDiff { // GetReelFirstRegularExpressions returns the regular expressions used for matching in ReelFirst. func (p *CnfFsDiff) GetReelFirstRegularExpressions() []string { - return []string{err, bin, sbin, lib, successfulOutputRegex, acceptAllRegex} + return []string{err, varlibrpm, varlibdpkg, bin, sbin, lib, usrbin, usrsbin, usrlib, successfulOutputRegex, acceptAllRegex} } From 0cfefc87e959cf42dd1146d878e2bd39bb34f332 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 2 Sep 2021 17:03:25 -0400 Subject: [PATCH 042/344] update documentation (#337) * update documentation * fix PR comments * fix pr comments * update doc * update 4.4.3 to 4.6.0 * fix Ying's comments * address david's comment * fix typo Co-authored-by: Jun Chen --- README.md | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a4bddbeb9..646d14a24 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,8 @@ The image can be pulled using : ```shell script docker pull quay.io/testnetworkfunction/test-network-function ``` +### Cluster requirement +To run all the tests, OCP cluster should provide enough resources to drain nodes and reschedule pods. If that's not the case, then ``lifecycle-pod-recreation`` test should be skipped. ### Check cluster resources Some tests suites such as platform-alteration require node access to get node configuration like hugepage. In order to get the required information, the test suite does not ssh into nodes, but instead rely on [oc debug tools ](https://docs.openshift.com/container-platform/3.7/cli_reference/basic_cli_operations.html#debug). This tool makes it easier to fetch information from nodes and also to debug running pods. @@ -236,16 +238,8 @@ To make `run-tnf-container.sh` use the newly built image, specify the custom TNF ## Building and running the standalone test executable -Currently, all available tests are part of the "CNF Certification Test Suite" test suite, which serves as the entrypoint -to run all test specs. `CNF Certification 3.0` is not containerized, and involves pulling, building, then running the -tests. - +Currently, all available tests are part of the "CNF Certification Test Suite" test suite, which serves as the entrypoint to run all test specs. By default, `test-network-function` emits results to `test-network-function/cnf-certification-tests_junit.xml`. - -The included default configuration is for running `generic` and `multus` suites on the trivial example at -[cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner). To configure for your -own environment, please see [config.md](docs/config.md). - ### Dependencies At a minimum, the following dependencies must be installed *prior* to running `make install-tools`. @@ -341,13 +335,13 @@ appropriate for the CNF(s) under test. Test suites group tests by topic area: Suite|Test Spec Description|Minimum OpenShift Version ---|---|--- -`access-control`|The access-control test suite is used to test service account, namespace and cluster/pod role binding for the pods under test. It also tests the pods/containers configuration.|4.4.3 -`affiliated-certification`|The affiliated-certification test suite verifies that the containers in the pod under test and operator under test are certified by Redhat|4.4.3 -`diagnostic`|The diagnostic test suite is used to gather node information from an OpenShift cluster. The diagnostic test suite should be run whenever generating a claim.json file.|4.4.3 -`lifecycle`| The lifecycle test suite verifies the pods deployment, creation, shutdown and survivability. |4.4.3 -`networking`|The networking test suite contains tests that check connectivity and networking config related best practices.|4.4.3 -`operator`|The operator test suite is designed to test basic Kubernetes Operator functionality.|4.4.3 -`platform-alteration`| verifies that key platform configuration is not modified by the CNF under test|4.4.3 +`access-control`|The access-control test suite is used to test service account, namespace and cluster/pod role binding for the pods under test. It also tests the pods/containers configuration.|4.6.0 +`affiliated-certification`|The affiliated-certification test suite verifies that the containers in the pod under test and operator under test are certified by Redhat|4.6.0 +`diagnostic`|The diagnostic test suite is used to gather node information from an OpenShift cluster. The diagnostic test suite should be run whenever generating a claim.json file.|4.6.0 +`lifecycle`| The lifecycle test suite verifies the pods deployment, creation, shutdown and survivability. |4.6.0 +`networking`|The networking test suite contains tests that check connectivity and networking config related best practices.|4.6.0 +`operator`|The operator test suite is designed to test basic Kubernetes Operator functionality.|4.6.0 +`platform-alteration`| verifies that key platform configuration is not modified by the CNF under test|4.6.0 Please consult [CATALOG.md](CATALOG.md) for a detailed description of tests in each suite. @@ -547,7 +541,7 @@ output. For example: ```shell script -TNF_DEFAULT_BUFFER_SIZE=32768 ./run-cnf-suites.sh diagnostic generic +TNF_DEFAULT_BUFFER_SIZE=32768 ./run-cnf-suites.sh -f diagnostic ``` ## Issue-161 Some containers under test do not contain `ping` or `ip` binary utilities From 16f477463907b7689895c08438ee6d55f5f3ccb2 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 3 Sep 2021 09:54:15 -0400 Subject: [PATCH 043/344] Update tests description (#343) --- CATALOG.md | 8 ++++---- test-network-function/identifiers/identifiers.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 1aec18284..23c8b1fe6 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -117,7 +117,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s). Result Type|normative -Suggested Remediation|Deploy the CNF using DaemonSet or ReplicaSet. +Suggested Remediation|Deploy the CNF using ReplicaSet. ### http://test-network-function.com/testcases/lifecycle/pod-recreation Property|Description @@ -165,7 +165,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative -Suggested Remediation|Ensure Services are not configured to not use NodePort(s). +Suggested Remediation|Ensure Services are not configured to use NodePort(s). ### http://test-network-function.com/testcases/operator/install-source Property|Description @@ -179,7 +179,7 @@ Suggested Remediation|Ensure that your Operator is installed via OLM. Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The Operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. +Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. Result Type|normative Suggested Remediation|Ensure that your Operator abides by the Operator Best Practices mentioned in the description. ### http://test-network-function.com/testcases/platform-alteration/base-image @@ -499,7 +499,7 @@ Runtime Binaries Required|`oc`, `cat`, `echo` Property|Description ---|--- Version|v1.0.0 -Description|An operator-specific test used to exercise the behavior of a given operator. In the current offering, we check if the operator ClusterServiceVersion (CSV) is installed properly. A CSV is a YAML manifest created from Operator metadata that assists the Operator Lifecycle Manager (OLM) in running the Operator. +Description|An operator-specific test used to exercise the behavior of a given operator. In the current offering, we check if the operator ClusterServiceVersion (CSV) is installed properly. A CSV is a YAML manifest created from Operator metadata that assists the Operator Lifecycle Manager (OLM) in running the Operator. Result Type|normative Intrusive|false Modifications Persist After Test|false diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index bdbdef48b..894a911c8 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -375,7 +375,7 @@ ClusterRoleBindings, if possible.`, TestPodDeploymentBestPracticesIdentifier: { Identifier: TestPodDeploymentBestPracticesIdentifier, Type: normativeResult, - Remediation: `Deploy the CNF using DaemonSet or ReplicaSet.`, + Remediation: `Deploy the CNF using ReplicaSet.`, Description: formDescription(TestPodDeploymentBestPracticesIdentifier, `tests that CNF Pod(s) are deployed as part of a ReplicaSet(s).`), }, @@ -399,7 +399,7 @@ ClusterRoleBindings, if possible.`, TestServicesDoNotUseNodeportsIdentifier: { Identifier: TestServicesDoNotUseNodeportsIdentifier, Type: normativeResult, - Remediation: `Ensure Services are not configured to not use NodePort(s).`, + Remediation: `Ensure Services are not configured to use NodePort(s).`, Description: formDescription(TestServicesDoNotUseNodeportsIdentifier, `tests that each CNF Service does not utilize NodePort(s).`), }, From e56c260b2c7f646f75be297e483627a629e72894 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 3 Sep 2021 16:23:00 +0200 Subject: [PATCH 044/344] Autodiscovery label field namespace renamed to prefix. (#340) * Autodiscovery label field namespace renamed to prefix. * README update. Co-authored-by: Jun Chen --- README.md | 4 ++-- pkg/config/autodiscover/autodiscover.go | 12 ++++++------ pkg/config/autodiscover/autodiscover_partner.go | 2 +- pkg/config/autodiscover/autodiscover_targets.go | 2 +- pkg/config/autodiscover/csv_info.go | 2 +- pkg/config/configsections/common.go | 6 +++--- test-network-function/tnf_config.yml | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 646d14a24..df0d92c5d 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ In the diagram above: The Test Network Function support auto-configuration using labels and annotations, but also a static configuration using a file. The following sections describe how to configure the TNF via labels/annotation and the corresponding settings in the config file. A sample config file can be found [here](test-network-function/tnf_config.yml). ### targetPodLabels -The goal of this section is to specify the label to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The namespace field is just a prefix for the label name, e.g. with the default configuration: +The goal of this section is to specify the label to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The prefix field can be used to avoid naming collision with other labels. ```shell script targetPodLabels: - - namespace: test-network-function.com + - prefix: test-network-function.com name: generic value: target ``` diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index ce35dddff..7db2ecbfe 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -28,7 +28,7 @@ import ( const ( disableAutodiscoverEnvVar = "TNF_DISABLE_CONFIG_AUTODISCOVER" - tnfNamespace = "test-network-function.com" + tnfLabelPrefix = "test-network-function.com" labelTemplate = "%s/%s" // anyLabelValue is the value that will allow any value for a label when building the label query. @@ -49,15 +49,15 @@ func buildLabelName(labelNS, labelName string) string { } func buildAnnotationName(annotationName string) string { - return buildLabelName(tnfNamespace, annotationName) + return buildLabelName(tnfLabelPrefix, annotationName) } func buildLabelQuery(label configsections.Label) string { - namespacedLabel := buildLabelName(label.Namespace, label.Name) + fullLabelName := buildLabelName(label.Prefix, label.Name) if label.Value != anyLabelValue { - return fmt.Sprintf("%s=%s", namespacedLabel, label.Value) + return fmt.Sprintf("%s=%s", fullLabelName, label.Value) } - return namespacedLabel + return fullLabelName } func makeGetCommand(resourceType, labelQuery string) *exec.Cmd { @@ -100,7 +100,7 @@ func getContainerByLabel(label configsections.Label) (container configsections.C return container, err } if len(containers) != 1 { - return container, fmt.Errorf("expected exactly one container, got %d for label %s/%s=%s", len(containers), label.Namespace, label.Name, label.Value) + return container, fmt.Errorf("expected exactly one container, got %d for label %s/%s=%s", len(containers), label.Prefix, label.Name, label.Value) } return containers[0], nil } diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index f35da8d23..1c66d91d0 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -30,7 +30,7 @@ const ( // using labels and annotations to populate the data, if it's not fully configured func FindTestPartner(tp *configsections.TestPartner) { if tp.TestOrchestratorID.ContainerName == "" { - orchestrator, err := getContainerByLabel(configsections.Label{Namespace: tnfNamespace, Name: genericLabelName, Value: orchestratorValue}) + orchestrator, err := getContainerByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: genericLabelName, Value: orchestratorValue}) if err != nil { log.Fatalf("failed to identify a single test orchestrator container: %s", err) } diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index b5f6de356..5b8a1c279 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -49,7 +49,7 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa } } // Containers to exclude from connectivity tests are optional - identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Namespace: tnfNamespace, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) target.ExcludeContainersFromConnectivityTests = identifiers if err != nil { diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index cca9eb544..a167927ba 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -69,7 +69,7 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error // GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that // label will be returned, regardless of the labels value. func GetCSVsByLabel(labelName, labelValue string) (*CSVList, error) { - cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Namespace: tnfNamespace, Name: labelName, Value: labelValue})) + cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue})) out, err := cmd.Output() if err != nil { diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index c88c50bd2..7722cac96 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -18,9 +18,9 @@ package configsections // Label ns/name/value for resource lookup type Label struct { - Namespace string `yaml:"namespace" json:"namespace"` - Name string `yaml:"name" json:"name"` - Value string `yaml:"value" json:"value"` + Prefix string `yaml:"prefix" json:"prefix"` + Name string `yaml:"name" json:"name"` + Value string `yaml:"value" json:"value"` } // Operator struct defines operator manifest for testing diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 946ea96d3..859506a1b 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -1,5 +1,5 @@ targetPodLabels: - - namespace: test-network-function.com + - prefix: test-network-function.com name: generic value: target # The following section does not require manual configuration as autodiscovery is on by default From 0550c16f9f3c183f982d320bdb76eb9380ecf408 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 3 Sep 2021 17:00:25 +0200 Subject: [PATCH 045/344] Added deployment list to autodiscover config. (#313) * Added deployment list to autodiscover config. * Unit Tests added to autodiscover/deployment_info.go * Use original targetPod labels. No exclusion labels needed. * Addressing Jun's comments. + DeploymentsUnderTest will be autodiscovered, but they can also be explicitly configured in the yaml config file, under the testTarget section. + Added Ginkgo Fail statement in some TCs when no deployments have been discovered. + Minor refactors. * README file updated. * Addressing Salah's comments. + TCs that use config's deployments list will be skipped instead of fail if that list is empty. * Removing manual config of the yaml file. * Last README fix to remove manual config info. * README file: restored sentece about the deployments list. * Changed namespace label field to prefix. --- README.md | 2 +- .../autodiscover/autodiscover_targets.go | 37 +++++++ pkg/config/autodiscover/deployment_info.go | 90 +++++++++++++++ .../autodiscover/deployment_info_test.go | 59 ++++++++++ .../autodiscover/testdata/testdeployment.json | 15 +++ pkg/config/config.go | 9 +- pkg/config/config_instance_test.go | 16 +++ pkg/config/configsections/common.go | 2 + .../configsections/config_section_test.go | 17 ++- pkg/config/configsections/deployment.go | 24 ++++ pkg/config/configsections/misc.go | 9 -- pkg/config/testdata/tnf_test_config.yml | 5 + test-network-function/lifecycle/suite.go | 104 ++++++------------ 13 files changed, 300 insertions(+), 89 deletions(-) create mode 100644 pkg/config/autodiscover/deployment_info.go create mode 100644 pkg/config/autodiscover/deployment_info_test.go create mode 100644 pkg/config/autodiscover/testdata/testdeployment.json create mode 100644 pkg/config/configsections/deployment.go diff --git a/README.md b/README.md index df0d92c5d..05a47f8fc 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The corresponding label used to match pods is: test-network-function.com/generic: target ``` -Once the pods are found, all of their containers are also added to the target container list. +Once the pods are found, all of their containers are also added to the target container list. A target deployments list will also be created with all the deployments which the test pods belong to. ### testTarget #### podsUnderTest / containersUnderTest diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 5b8a1c279..c76cd6b0e 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -64,6 +64,43 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa } else { log.Warnf("an error (%s) occurred when looking for operaters by label", err) } + + target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target)...) +} + +// FindTestDeployments uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, +// currently partner and fs_diff ones. +func FindTestDeployments(targetLabels []configsections.Label, target *configsections.TestTarget) (deployments []configsections.Deployment) { + namespaces := make(map[string]bool) + + for _, cut := range target.ContainerConfigList { + namespace := cut.ContainerIdentifier.Namespace + if _, alreadyProcessed := namespaces[namespace]; alreadyProcessed { + continue + } + + namespaces[namespace] = true + + for _, label := range targetLabels { + deploymentResourceList, err := GetTargetDeploymentsByNamespace(namespace, label) + + if err != nil { + log.Error("Unable to get deployment list from namespace ", namespace, ". Error: ", err) + } else { + for _, deploymentResource := range deploymentResourceList.Items { + deployment := configsections.Deployment{ + Name: deploymentResource.GetName(), + Namespace: deploymentResource.GetNamespace(), + Replicas: deploymentResource.GetReplicas(), + } + + deployments = append(deployments, deployment) + } + } + } + } + + return deployments } // buildPodUnderTest builds a single `configsections.Pod` from a PodResource diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go new file mode 100644 index 000000000..2e4f8a834 --- /dev/null +++ b/pkg/config/autodiscover/deployment_info.go @@ -0,0 +1,90 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "encoding/json" + "fmt" + "os/exec" + + "github.com/test-network-function/test-network-function/pkg/config/configsections" +) + +const ( + resourceTypeDeployment = "deployment" +) + +// DeploymentList holds the data from an `oc get deployments -o json` command +type DeploymentList struct { + Items []DeploymentResource `json:"items"` +} + +// DeploymentResource defines deployment resources +type DeploymentResource struct { + Metadata struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` + + Spec struct { + Replicas int `json:"replicas"` + } +} + +// GetName returns deployment's metadata section's name field. +func (deployment *DeploymentResource) GetName() string { + return deployment.Metadata.Name +} + +// GetNamespace returns deployment's metadata section's namespace field. +func (deployment *DeploymentResource) GetNamespace() string { + return deployment.Metadata.Namespace +} + +// GetReplicas returns deployment's spec section's replicas field. +func (deployment *DeploymentResource) GetReplicas() int { + return deployment.Spec.Replicas +} + +// GetLabels returns a map with the deployment's metadata section's labels. +func (deployment *DeploymentResource) GetLabels() map[string]string { + return deployment.Metadata.Labels +} + +// GetTargetDeploymentsByNamespace will return all deployments that have pods with a given label. +func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsections.Label) (*DeploymentList, error) { + labelQuery := fmt.Sprintf("\"%s/%s\"==\"%s\"", targetLabel.Prefix, targetLabel.Name, targetLabel.Value) + jqArgs := fmt.Sprintf("'[.items[] | select(.spec.template.metadata.labels.%s)]'", labelQuery) + ocCmd := fmt.Sprintf("oc get %s -n %s -o json | jq %s", resourceTypeDeployment, namespace, jqArgs) + + cmd := exec.Command("bash", "-c", ocCmd) + + out, err := cmd.Output() + if err != nil { + return nil, err + } + + var deploymentList DeploymentList + err = json.Unmarshal(out, &deploymentList.Items) + if err != nil { + return nil, err + } + + return &deploymentList, nil +} diff --git a/pkg/config/autodiscover/deployment_info_test.go b/pkg/config/autodiscover/deployment_info_test.go new file mode 100644 index 000000000..b15a0b8b0 --- /dev/null +++ b/pkg/config/autodiscover/deployment_info_test.go @@ -0,0 +1,59 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "encoding/json" + "io/ioutil" + "log" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + testDeploymentFile = "testdeployment.json" +) + +var ( + testDeploymentFilePath = path.Join(filePath, testDeploymentFile) +) + +func loadDeployment(filePath string) (deployment DeploymentResource) { + contents, err := ioutil.ReadFile(filePath) + if err != nil { + log.Fatalf("error (%s) loading DeploymentResource %s for testing", err, filePath) + } + err = json.Unmarshal(contents, &deployment) + if err != nil { + log.Fatalf("error (%s) unmarshalling DeploymentResource %s for testing", err, filePath) + } + return +} + +func TestPodGetAnnotationValue1(t *testing.T) { + deployment := loadDeployment(testDeploymentFilePath) + + assert.Equal(t, "test", deployment.GetName()) + assert.Equal(t, "tnf", deployment.GetNamespace()) + assert.Equal(t, 2, deployment.GetReplicas()) + + labels := deployment.GetLabels() + assert.Equal(t, 1, len(labels)) + assert.Equal(t, "test", labels["app"]) +} diff --git a/pkg/config/autodiscover/testdata/testdeployment.json b/pkg/config/autodiscover/testdata/testdeployment.json new file mode 100644 index 000000000..63968b858 --- /dev/null +++ b/pkg/config/autodiscover/testdata/testdeployment.json @@ -0,0 +1,15 @@ + +{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app": "test" + }, + "name": "test", + "namespace": "tnf" + }, + "spec": { + "replicas": 2 + } +} \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go index 48c6881cf..7c9d8b5b8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -112,9 +112,10 @@ func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) string // TestEnvironment includes the representation of the current state of the test targets and parters as well as the test configuration type TestEnvironment struct { - ContainersUnderTest map[configsections.ContainerIdentifier]*Container - PartnerContainers map[configsections.ContainerIdentifier]*Container - PodsUnderTest []configsections.Pod + ContainersUnderTest map[configsections.ContainerIdentifier]*Container + PartnerContainers map[configsections.ContainerIdentifier]*Container + PodsUnderTest []configsections.Pod + DeploymentsUnderTest []configsections.Deployment // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -180,6 +181,8 @@ func (env *TestEnvironment) doAutodiscover() { env.PodsUnderTest = env.Config.PodsUnderTest env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] + env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest + log.Info(env.TestOrchestrator) log.Info(env.ContainersUnderTest) env.needsRefresh = false diff --git a/pkg/config/config_instance_test.go b/pkg/config/config_instance_test.go index 0d79343b8..a9efbd8f2 100644 --- a/pkg/config/config_instance_test.go +++ b/pkg/config/config_instance_test.go @@ -20,12 +20,27 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/config/configsections" ) const ( filePath = "testdata/tnf_test_config.yml" ) +const ( + testDeploymentsNumber = 1 + testDeploymentName = "test" + testDeploymentNamespace = "default" + testDeploymentReplicas = 2 +) + +func testLoadedDeployments(t *testing.T, deployments []configsections.Deployment) { + assert.Equal(t, len(deployments), testDeploymentsNumber) + assert.Equal(t, deployments[0].Name, testDeploymentName) + assert.Equal(t, deployments[0].Namespace, testDeploymentNamespace) + assert.Equal(t, deployments[0].Replicas, testDeploymentReplicas) +} + func TestLoadConfigFromFile(t *testing.T) { env := GetTestEnvironment() assert.Nil(t, env.loadConfigFromFile(filePath)) @@ -33,4 +48,5 @@ func TestLoadConfigFromFile(t *testing.T) { assert.Equal(t, env.Config.Partner.TestOrchestratorID.Namespace, "default") assert.Equal(t, env.Config.Partner.TestOrchestratorID.ContainerName, "partner") assert.Equal(t, env.Config.Partner.TestOrchestratorID.PodName, "partner") + testLoadedDeployments(t, env.Config.DeploymentsUnderTest) } diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 7722cac96..b9c740b34 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -64,6 +64,8 @@ type TestPartner struct { // TestTarget is a collection of resources under test type TestTarget struct { + // DeploymentsUnderTest is the list of deployments that contain pods under test. + DeploymentsUnderTest []Deployment `yaml:"deploymentsUnderTest" json:"deploymentsUnderTest"` // PodsUnderTest is the list of the pods that needs to be tested. Each entry is a single pod to be tested. PodsUnderTest []Pod `yaml:"podsUnderTest,omitempty" json:"podsUnderTest,omitempty"` // ContainerConfigList is the list of containers that needs to be tested. diff --git a/pkg/config/configsections/config_section_test.go b/pkg/config/configsections/config_section_test.go index 9131e0670..2d34e4f19 100644 --- a/pkg/config/configsections/config_section_test.go +++ b/pkg/config/configsections/config_section_test.go @@ -47,7 +47,7 @@ const ( // deploymentName is the name of the deployment deploymentName = "deployment-one" // deploymentReplicas no of replicas - deploymentReplicas = "1" + deploymentReplicas = 1 // fullConfig represents full configuration, including Operator and CNF fullConfig = "full_config" // instanceNameOne name of the instance @@ -105,6 +105,16 @@ func newConfig(configPath string) (*TestConfiguration, error) { return conf, nil } +func loadDeploymentsConfig() { + test.DeploymentsUnderTest = []Deployment{ + { + Name: deploymentName, + Namespace: testNameSpace, + Replicas: deploymentReplicas, + }, + } +} + func loadPodConfig() { test.PodsUnderTest = []Pod{ { @@ -120,9 +130,6 @@ func loadOperatorConfig() { operator.Name = operatorName operator.Namespace = operatorNameSpace setCrdsAndInstances() - dep := Deployment{} - dep.Name = deploymentName - dep.Replicas = deploymentReplicas operator.Tests = []string{testcases.OperatorStatus} test.Operators = append(test.Operators, operator) loadPodConfig() @@ -146,6 +153,7 @@ func setCrdsAndInstances() { func loadFullConfig() { loadOperatorConfig() loadPodConfig() + loadDeploymentsConfig() } func setup(configType string) { @@ -159,6 +167,7 @@ func setup(configType string) { loadFullConfig() case cnfConfig: loadPodConfig() + loadDeploymentsConfig() case operatorConfig: loadOperatorConfig() } diff --git a/pkg/config/configsections/deployment.go b/pkg/config/configsections/deployment.go new file mode 100644 index 000000000..31d698a31 --- /dev/null +++ b/pkg/config/configsections/deployment.go @@ -0,0 +1,24 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +// Deployment defines a deployment in the cluster. +type Deployment struct { + Name string + Namespace string + Replicas int +} diff --git a/pkg/config/configsections/misc.go b/pkg/config/configsections/misc.go index 001f82e54..276116724 100644 --- a/pkg/config/configsections/misc.go +++ b/pkg/config/configsections/misc.go @@ -34,15 +34,6 @@ type Crd struct { Instances []Instance `yaml:"instances" json:"instances"` } -// Deployment defines deployment resources -type Deployment struct { - // Name is the name of the deployment specified in the CSV - Name string `yaml:"name" json:"name"` - - // Replicas is no of replicas that are expected for this deployment as specified in the CSV - Replicas string `yaml:"replicas" json:"replicas"` -} - // Permission defines roles and cluster roles resources type Permission struct { // Name is the name of Roles and Cluster Roles that is specified in the CSV diff --git a/pkg/config/testdata/tnf_test_config.yml b/pkg/config/testdata/tnf_test_config.yml index 2867e0014..69f3e2e22 100644 --- a/pkg/config/testdata/tnf_test_config.yml +++ b/pkg/config/testdata/tnf_test_config.yml @@ -24,6 +24,11 @@ testTarget: tests: - PRIVILEGED_POD - PRIVILEGED_ROLE + deploymentsUnderTest: + - name: test + namespace: default + replicas: 2 + testPartner: partnerContainers: - namespace: default diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 93e2c7ba7..012696551 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -20,10 +20,10 @@ import ( "fmt" "path" "sort" - "strings" "time" "github.com/test-network-function/test-network-function/pkg/config" + "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -49,7 +49,6 @@ import ( const ( defaultTerminationGracePeriod = 30 drainTimeoutMinutes = 5 - partnerPod = "partner" scalingTimeout = 60 * time.Second scalingPollingPeriod = 1 * time.Second ) @@ -121,54 +120,36 @@ func waitForAllDeploymentsReady(namespace string, timeout, pollingPeriod time.Du } // restoreDeployments is the last attempt to restore the original test deployments' replicaCount -func restoreDeployments(env *config.TestEnvironment, nsDeployments *map[string]dp.DeploymentMap) { - for namespace, originalDeployments := range *nsDeployments { - // For each deployment in the namespace, get the current replicas and compare. - deployments, notReadyDeployments := getDeployments(namespace) +func restoreDeployments(env *config.TestEnvironment) { + for _, deployment := range env.DeploymentsUnderTest { + // For each test deployment in the namespace, refresh the current replicas and compare. + deployments, notReadyDeployments := getDeployments(deployment.Namespace) if len(notReadyDeployments) > 0 { // Wait until the deployment is ready - waitForAllDeploymentsReady(namespace, scalingTimeout, scalingPollingPeriod) + waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } - for originalDeploymentName, originalDeployment := range originalDeployments { - deployment := deployments[originalDeploymentName] - - if deployment.Replicas == originalDeployment.Replicas { - continue - } + if deployments[deployment.Name].Replicas != deployment.Replicas { + log.Warn("Deployment ", deployment.Name, " replicaCount (", deployment.Replicas, ") needs to be restored.") // Try to scale to the original deployment's replicaCount. - runScalingTest(namespace, originalDeploymentName, originalDeployment.Replicas) + runScalingTest(deployment) env.SetNeedsRefresh() } } } -// saveDeployment Stores the dp.Deployment data into a map of namespace -> deployments -func saveDeployment(nsDeployments map[string]dp.DeploymentMap, namespace, deploymentName string, deployment *dp.Deployment) { - deployments, namespaceExists := nsDeployments[namespace] - - if !namespaceExists { - deployments = dp.DeploymentMap{} - deployments[deploymentName] = *deployment - nsDeployments[namespace] = deployments - } else { - // In case the deploymentName already exists, it will be overwritten. - deployments[deploymentName] = *deployment - } -} - -// runScalingTest Runs a Scaling handler TC with a given replicaCount and waits for all the deployments to be ready. -func runScalingTest(namespace, deploymentName string, replicaCount int) { - handler := scaling.NewScaling(common.DefaultTimeout, namespace, deploymentName, replicaCount) +// runScalingTest Runs a Scaling handler TC and waits for all the deployments to be ready. +func runScalingTest(deployment configsections.Deployment) { + handler := scaling.NewScaling(common.DefaultTimeout, deployment.Namespace, deployment.Name, deployment.Replicas) test, err := tnf.NewTest(common.GetContext().GetExpecter(), handler, []reel.Handler{handler}, common.GetContext().GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) common.RunAndValidateTest(test) // Wait until the deployment is ready - waitForAllDeploymentsReady(namespace, scalingTimeout, scalingPollingPeriod) + waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } func testScaling(env *config.TestEnvironment) { @@ -176,45 +157,28 @@ func testScaling(env *config.TestEnvironment) { ginkgo.It(testID, func() { ginkgo.By("Testing deployment scaling") defer results.RecordResult(identifiers.TestScalingIdentifier) + defer restoreDeployments(env) - namespaceDeploymentsBackup := make(map[string]dp.DeploymentMap) - defer restoreDeployments(env, &namespaceDeploymentsBackup) - - // Map to register the deployments that have been already tested - deploymentNames := make(map[string]bool) - - for _, podUnderTest := range env.PodsUnderTest { - podName := podUnderTest.Name - namespace := podUnderTest.Namespace - - // Get deployment name and check whether it was already tested. - // ToDo: Proper way (helper/handler) to do this. - podNameParts := strings.Split(podName, "-") - deploymentName := podNameParts[0] - msg := fmt.Sprintf("Testing deployment=%s, namespace=%s pod name=%s", deploymentName, namespace, podName) - log.Info(msg) - if _, alreadyTested := deploymentNames[deploymentName]; alreadyTested { - continue - } + if len(env.DeploymentsUnderTest) == 0 { + ginkgo.Skip("No test deployments found.") + } - // Save deployment data for deferred restoring in case something's wrong during the TC. - deployments, _ := getDeployments(namespace) - deployment := deployments[deploymentName] - saveDeployment(namespaceDeploymentsBackup, namespace, deploymentName, &deployment) + for _, deployment := range env.DeploymentsUnderTest { + ginkgo.By(fmt.Sprintf("Scaling Deployment=%s, Replicas=%d (ns=%s)", + deployment.Name, deployment.Replicas, deployment.Namespace)) replicaCount := deployment.Replicas // ScaleIn, removing one pod from the replicaCount - runScalingTest(namespace, deploymentName, (replicaCount - 1)) + deployment.Replicas = replicaCount - 1 + runScalingTest(deployment) // Scaleout, restoring the original replicaCount number - runScalingTest(namespace, deploymentName, replicaCount) + deployment.Replicas = replicaCount + runScalingTest(deployment) // Ensure next tests/test suites receive a refreshed config. env.SetNeedsRefresh() - - // Set this deployment as tested - deploymentNames[deploymentName] = true } }) } @@ -434,23 +398,19 @@ func uncordonNode(node string) { // Pod antiaffinity test for all deployments func testPodAntiAffinity(env *config.TestEnvironment) { - var deployments dp.DeploymentMap ginkgo.When("CNF is designed in high availability mode ", func() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodHighAvailabilityBestPractices) ginkgo.It(testID, func() { ginkgo.By("Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment") - for _, podUnderTest := range env.PodsUnderTest { - podNamespace := podUnderTest.Namespace + if len(env.DeploymentsUnderTest) == 0 { + ginkgo.Skip("No test deployments found.") + } + + for _, deployment := range env.DeploymentsUnderTest { defer results.RecordResult(identifiers.TestPodHighAvailabilityBestPractices) - deployments, _ = getDeployments(podNamespace) - if len(deployments) == 0 { - return - } - for name, d := range deployments { - if name != partnerPod { - podAntiAffinity(name, podNamespace, d.Replicas) - } - } + ginkgo.By(fmt.Sprintf("Testing Pod AntiAffinity on Deployment=%s, Replicas=%d (ns=%s)", + deployment.Name, deployment.Replicas, deployment.Namespace)) + podAntiAffinity(deployment.Name, deployment.Namespace, deployment.Replicas) } }) }) From 9d2b65d5de1b5a46ece935520d1cfa84ec4ffe12 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 3 Sep 2021 11:16:02 -0400 Subject: [PATCH 046/344] Fix Typo in boot-param description (#344) Co-authored-by: Jun Chen --- CATALOG.md | 2 +- test-network-function/identifiers/identifiers.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 23c8b1fe6..07c47b3df 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -197,7 +197,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/boot-params tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node. Result Type|normative -Suggested Remediation|Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerfromanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you. +Suggested Remediation|Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerformanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you. ### http://test-network-function.com/testcases/platform-alteration/hugepages-config Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 894a911c8..e3adf55fe 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -438,8 +438,7 @@ that there are no changes to the following directories: TestUnalteredStartupBootParamsIdentifier: { Identifier: TestUnalteredStartupBootParamsIdentifier, Type: normativeResult, - Remediation: `Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the -PerfromanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage + Remediation: `Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerformanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you.`, Description: formDescription(TestUnalteredStartupBootParamsIdentifier, `tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node.`), From 76d17d591c7e552a9248be45ee3271f83ace8de8 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Tue, 7 Sep 2021 16:06:08 -0500 Subject: [PATCH 047/344] Adding operator testing support (#342) * Adding operator testing support * pointing to official partner test repo * back to test branch * point again to main branch Co-authored-by: Jun Chen --- .github/workflows/pre-main.yaml | 2 ++ pkg/config/autodiscover/autodiscover_targets.go | 8 ++++---- pkg/config/autodiscover/csv_info.go | 6 +++++- pkg/config/autodiscover/testdata/csv.json | 3 ++- test-network-function/operator/suite.go | 6 ------ 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 8d1fafca8..9e6430787 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -111,6 +111,8 @@ jobs: - name: Start the minikube cluster for `local-test-infra` uses: ./cnf-certification-test-partner/.github/actions/start-minikube + with: + working_directory: cnf-certification-test-partner - name: Create `local-test-infra` OpenShift resources uses: ./cnf-certification-test-partner/.github/actions/create-local-test-infra-resources diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index c76cd6b0e..3caac4b56 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -136,13 +136,13 @@ func buildOperatorFromCSVResource(csv *CSVResource) (op configsections.Operator) op.Tests = tests } - var subscriptionName string + var subscriptionName []string err = csv.GetAnnotationValue(subscriptionNameAnnotationName, &subscriptionName) if err != nil { - log.Warnf("unable to get a subscription name annotation from CSV %s (%s), the CSV name will be used", csv.Metadata.Name, err) + log.Warnf("unable to get a subscription name annotation from CSV %s (error: %s).", csv.Metadata.Name, err) + } else { + op.SubscriptionName = subscriptionName[0] } - op.SubscriptionName = subscriptionName - return op } diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index a167927ba..e2aafb77c 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/config/configsections" ) @@ -75,7 +77,9 @@ func GetCSVsByLabel(labelName, labelValue string) (*CSVList, error) { if err != nil { return nil, err } - + log.Debug("JSON output for all pods labeled with: ", labelName) + log.Debug("Command: ", cmd) + log.Debug(string(out)) var csvList CSVList err = json.Unmarshal(out, &csvList) if err != nil { diff --git a/pkg/config/autodiscover/testdata/csv.json b/pkg/config/autodiscover/testdata/csv.json index f9a029334..d16c71295 100644 --- a/pkg/config/autodiscover/testdata/csv.json +++ b/pkg/config/autodiscover/testdata/csv.json @@ -1,7 +1,8 @@ { "metadata": { "annotations": { - "test-network-function.com/operator_tests": "[\"OPERATOR_STATUS\", \"ANOTHER_TEST\"]" + "test-network-function.com/operator_tests": "[\"OPERATOR_STATUS\", \"ANOTHER_TEST\"]", + "test-network-function.com/subscription_name": "[\"nginx-operator-v0-0-1-sub\"]" }, "labels": { "test-network-function.com/operator": "target" diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 9d574899a..2af9f3a7f 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -30,7 +30,6 @@ import ( "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -132,11 +131,6 @@ func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsection } func itRunsTestsOnOperator() { - if common.IsMinikube() { - log.Info("Minikube detected: skipping operators test.") - return - } - for _, testType := range testcases.GetConfiguredOperatorTests() { testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) gomega.Expect(testFile).ToNot(gomega.BeNil()) From e45c95d77082e1aeefda6193a39a76f5ffaec1d5 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 8 Sep 2021 11:54:12 -0400 Subject: [PATCH 048/344] Fix workflow to push new images to quay.io (#348) --- .github/workflows/tnf-image.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 6e7bfe7e8..ccdb5ef9e 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -115,10 +115,7 @@ jobs: shell: bash - name: 'Test: Run diagnostic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} diagnostic - - - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} generic + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic # Push the new TNF image to Quay.io. From 100705085d2853e2dbe1768cf67b26c34e120658 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 8 Sep 2021 12:06:25 -0400 Subject: [PATCH 049/344] Update tnf-image.yaml --- .github/workflows/tnf-image.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index ccdb5ef9e..5ea04ff71 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -100,7 +100,9 @@ jobs: - name: Start the minikube cluster for `local-test-infra` uses: ./cnf-certification-test-partner/.github/actions/start-minikube - + with: + working_directory: cnf-certification-test-partner + - name: Create `local-test-infra` OpenShift resources uses: ./cnf-certification-test-partner/.github/actions/create-local-test-infra-resources with: From 7192c5123c5f9f4c1f4357370f376cdb5927ca7b Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 8 Sep 2021 12:41:54 -0400 Subject: [PATCH 050/344] Update tnf-image.yaml --- .github/workflows/tnf-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 5ea04ff71..79886b41b 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -20,7 +20,7 @@ env: REGISTRY_LOCAL: localhost IMAGE_NAME: testnetworkfunction/test-network-function IMAGE_TAG: latest - CURRENT_VERSION_GENERIC_BRANCH: 2.0.x + CURRENT_VERSION_GENERIC_BRANCH: 3.0.x TNF_CONTAINER_CLIENT: docker TNF_MINIKUBE_ONLY: true TNF_NON_INTRUSIVE_ONLY: true From 795b91a27acb06226cab5b530029d1f0eb4d041a Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 8 Sep 2021 15:04:09 -0400 Subject: [PATCH 051/344] Expose cnf feature deploy test versoin and skip arg through env vars (#345) --- README.md | 13 ++----------- script/run-cfd-container.sh | 7 ++++--- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 05a47f8fc..006de93c5 100644 --- a/README.md +++ b/README.md @@ -139,19 +139,10 @@ The test suites from openshift-kni/cnf-feature-deploy can be run prior to the ac ```shell script -export VERIFY_CNF_FEATURES=true +export TNF_RUN_CFD_TEST=true ``` -Currently, these suites are skipped: -* performance -* sriov -* ptp -* sctp -* xt_u32 -* dpdk -* ovn - -For more information on the test suites, refer to [the cnf-features-deploy repository](https://github.com/openshift-kni/cnf-features-deploy/tree/release-4.6) +By default, the image with release tag `4.6` is used and the ginkgo skip argument is set to `performance|sriov|ptp|sctp|xt_u32|dpdk|ovn`. To override the default behavior, set these environment variables: `TNF_CFD_IMAGE_TAG` and `TNF_CFD_SKIP`. For more information on the test suites, refer to [the cnf-features-deploy repository](https://github.com/openshift-kni/cnf-features-deploy) ## Running the tests with in a prebuild container diff --git a/script/run-cfd-container.sh b/script/run-cfd-container.sh index 6d5b45fa3..10812104b 100755 --- a/script/run-cfd-container.sh +++ b/script/run-cfd-container.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -if [ "$VERIFY_CNF_FEATURES" == "true" ] && [ "$TNF_MINIKUBE_ONLY" != "true" ]; then +if [ "$TNF_RUN_CFD_TEST" == "true" ] && [ "$TNF_MINIKUBE_ONLY" != "true" ]; then export TNF_IMAGE_NAME=cnf-tests - export TNF_IMAGE_TAG=4.6 + export TNF_IMAGE_TAG="${TNF_CFD_IMAGE_TAG:-4.6}" export TNF_OFFICIAL_ORG=quay.io/openshift-kni/ export TNF_OFFICIAL_IMAGE="${TNF_OFFICIAL_ORG}${TNF_IMAGE_NAME}:${TNF_IMAGE_TAG}" + export TNF_CFD_SKIP="${TNF_CFD_SKIP:-performance|sriov|ptp|sctp|xt_u32|dpdk|ovn}" export TNF_CMD="/usr/bin/test-run.sh" export OUTPUT_ARG="--junit" export CONTAINER_NETWORK_MODE="host" @@ -19,7 +20,7 @@ if [ "$VERIFY_CNF_FEATURES" == "true" ] && [ "$TNF_MINIKUBE_ONLY" != "true" ]; t # For older verions of docker, dns server may need to be set explicitly, e.g. # # export DNS_ARG=172.0.0.53 - ./run-container.sh -ginkgo.v -ginkgo.skip="performance|sriov|ptp|sctp|xt_u32|dpdk|ovn" + ./run-container.sh -ginkgo.v -ginkgo.skip=${TNF_CFD_SKIP} else # removing report if not running, so the final claim won't include stale reports rm -f ${OUTPUT_LOC}/validation_junit.xml From 7d9eb483b7819b68421d442356d3b7d1acd6c390 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 8 Sep 2021 14:33:26 -0500 Subject: [PATCH 052/344] temporarily removing operator tests (#350) Co-authored-by: Salaheddine Hamadi --- .github/workflows/pre-main.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 9e6430787..bae7a8e41 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -125,8 +125,7 @@ jobs: run: ./run-cnf-suites.sh --focus diagnostic - name: 'Test: Run test suites' - run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification operator - + run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification # Perform smoke tests using a TNF container. - name: Build the `test-network-function` image @@ -149,7 +148,7 @@ jobs: run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification # Push the new unstable TNF image to Quay.io. From 6237a84cf37970a5199e9154510d55213029c6d7 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 8 Sep 2021 16:15:09 -0400 Subject: [PATCH 053/344] add namespace support (#347) * add namespace support skip networking tests if no orchestrator is found * Add namespace to Readme * update readme --- README.md | 5 +- pkg/config/autodiscover/autodiscover.go | 16 +++--- .../autodiscover/autodiscover_partner.go | 7 +-- .../autodiscover/autodiscover_targets.go | 49 +++++++------------ pkg/config/autodiscover/csv_info.go | 4 +- pkg/config/autodiscover/pod_info.go | 4 +- pkg/config/config.go | 11 +++-- pkg/config/configsections/common.go | 8 +++ test-network-function/networking/suite.go | 6 +++ test-network-function/tnf_config.yml | 2 + 10 files changed, 62 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 006de93c5..5beea9e8b 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,11 @@ In the diagram above: ## Test Configuration -The Test Network Function support auto-configuration using labels and annotations, but also a static configuration using a file. The following sections describe how to configure the TNF via labels/annotation and the corresponding settings in the config file. A sample config file can be found [here](test-network-function/tnf_config.yml). +The Test Network Function support autodiscovery using labels and annotations. The following sections describe how to configure the TNF via labels/annotation and the corresponding settings in the config file. A sample config file can be found [here](test-network-function/tnf_config.yml). +### targetNameSpaces + +A single namespace should be specified in the [configuration file](test-network-function/tnf_config.yml). This namespace will be used by autodiscovery to find the Pods under test. To run multiple tests in different namespaces simultaneously, intrusive tests should be disabled by setting ``TNF_NON_INTRUSIVE_ONLY`` to true. ### targetPodLabels The goal of this section is to specify the label to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The prefix field can be used to avoid naming collision with other labels. ```shell script diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 7db2ecbfe..d49d7d871 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -60,17 +60,17 @@ func buildLabelQuery(label configsections.Label) string { return fullLabelName } -func makeGetCommand(resourceType, labelQuery string) *exec.Cmd { +func makeGetCommand(resourceType, labelQuery, namespace string) *exec.Cmd { // TODO: shell expecter - cmd := exec.Command("oc", "get", resourceType, "-A", "-o", "json", "-l", labelQuery) + cmd := exec.Command("oc", "get", resourceType, "-n", namespace, "-o", "json", "-l", labelQuery) log.Debug("Issuing get command ", cmd.Args) return cmd } // getContainersByLabel builds `config.Container`s from containers in pods matching a label. -func getContainersByLabel(label configsections.Label) (containers []configsections.ContainerConfig, err error) { - pods, err := GetPodsByLabel(label) +func getContainersByLabel(label configsections.Label, namespace string) (containers []configsections.ContainerConfig, err error) { + pods, err := GetPodsByLabel(label, namespace) if err != nil { return nil, err } @@ -81,8 +81,8 @@ func getContainersByLabel(label configsections.Label) (containers []configsectio } // getContainerIdentifiersByLabel builds `config.ContainerIdentifier`s from containers in pods matching a label. -func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs []configsections.ContainerIdentifier, err error) { - containers, err := getContainersByLabel(label) +func getContainerIdentifiersByLabel(label configsections.Label, namespace string) (containerIDs []configsections.ContainerIdentifier, err error) { + containers, err := getContainersByLabel(label, namespace) if err != nil { return nil, err } @@ -94,8 +94,8 @@ func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs [] // getContainerByLabel returns exactly one container with the given label. If any other number of containers is found // then an error is returned along with an empty `config.Container`. -func getContainerByLabel(label configsections.Label) (container configsections.ContainerConfig, err error) { - containers, err := getContainersByLabel(label) +func getContainerByLabel(label configsections.Label, namespace string) (container configsections.ContainerConfig, err error) { + containers, err := getContainersByLabel(label, namespace) if err != nil { return container, err } diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index 1c66d91d0..04be8de75 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -28,11 +28,12 @@ const ( // FindTestPartner completes a `configsections.TestPartner` from the current state of the cluster, // using labels and annotations to populate the data, if it's not fully configured -func FindTestPartner(tp *configsections.TestPartner) { +func FindTestPartner(tp *configsections.TestPartner, namespace string) { if tp.TestOrchestratorID.ContainerName == "" { - orchestrator, err := getContainerByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: genericLabelName, Value: orchestratorValue}) + orchestrator, err := getContainerByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: genericLabelName, Value: orchestratorValue}, namespace) if err != nil { - log.Fatalf("failed to identify a single test orchestrator container: %s", err) + log.Errorf("failed to identify a single test orchestrator container: %s", err) + return } tp.ContainerConfigList = append(tp.ContainerConfigList, orchestrator) tp.TestOrchestratorID = orchestrator.ContainerIdentifier diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 3caac4b56..419aaa31e 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -35,10 +35,10 @@ var ( // FindTestTarget finds test targets from the current state of the cluster, // using labels and annotations, and add them to the `configsections.TestTarget` passed in. -func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget) { +func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget, namespace string) { // find pods by label for _, l := range labels { - pods, err := GetPodsByLabel(l) + pods, err := GetPodsByLabel(l, namespace) if err == nil { for i := range pods.Items { target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(&pods.Items[i])) @@ -49,14 +49,14 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa } } // Containers to exclude from connectivity tests are optional - identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}, namespace) target.ExcludeContainersFromConnectivityTests = identifiers if err != nil { log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) } - csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue) + csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue, namespace) if err == nil { for i := range csvs.Items { target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csvs.Items[i])) @@ -65,41 +65,28 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa log.Warnf("an error (%s) occurred when looking for operaters by label", err) } - target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target)...) + target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target, namespace)...) } // FindTestDeployments uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, // currently partner and fs_diff ones. -func FindTestDeployments(targetLabels []configsections.Label, target *configsections.TestTarget) (deployments []configsections.Deployment) { - namespaces := make(map[string]bool) - - for _, cut := range target.ContainerConfigList { - namespace := cut.ContainerIdentifier.Namespace - if _, alreadyProcessed := namespaces[namespace]; alreadyProcessed { - continue - } - - namespaces[namespace] = true - - for _, label := range targetLabels { - deploymentResourceList, err := GetTargetDeploymentsByNamespace(namespace, label) - - if err != nil { - log.Error("Unable to get deployment list from namespace ", namespace, ". Error: ", err) - } else { - for _, deploymentResource := range deploymentResourceList.Items { - deployment := configsections.Deployment{ - Name: deploymentResource.GetName(), - Namespace: deploymentResource.GetNamespace(), - Replicas: deploymentResource.GetReplicas(), - } - - deployments = append(deployments, deployment) +func FindTestDeployments(targetLabels []configsections.Label, target *configsections.TestTarget, namespace string) (deployments []configsections.Deployment) { + for _, label := range targetLabels { + deploymentResourceList, err := GetTargetDeploymentsByNamespace(namespace, label) + if err != nil { + log.Error("Unable to get deployment list from namespace ", namespace, ". Error: ", err) + } else { + for _, deploymentResource := range deploymentResourceList.Items { + deployment := configsections.Deployment{ + Name: deploymentResource.GetName(), + Namespace: deploymentResource.GetNamespace(), + Replicas: deploymentResource.GetReplicas(), } + + deployments = append(deployments, deployment) } } } - return deployments } diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index e2aafb77c..e318df4ff 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -70,8 +70,8 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error // GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that // label will be returned, regardless of the labels value. -func GetCSVsByLabel(labelName, labelValue string) (*CSVList, error) { - cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue})) +func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { + cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) out, err := cmd.Output() if err != nil { diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index fe0cc49c2..b2bd61581 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -151,8 +151,8 @@ func (pr *PodResource) annotationUnmarshalError(annotationKey string, err error) // GetPodsByLabel will return all pods with a given label value. If `labelValue` is an empty string, all pods with that // label will be returned, regardless of the labels value. -func GetPodsByLabel(label configsections.Label) (*PodList, error) { - cmd := makeGetCommand(resourceTypePods, buildLabelQuery(label)) +func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, error) { + cmd := makeGetCommand(resourceTypePods, buildLabelQuery(label), namespace) out, err := cmd.Output() if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 7c9d8b5b8..c75b5a1b4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -116,6 +116,7 @@ type TestEnvironment struct { PartnerContainers map[configsections.ContainerIdentifier]*Container PodsUnderTest []configsections.Pod DeploymentsUnderTest []configsections.Deployment + NameSpaceUnderTest string // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -160,15 +161,20 @@ func (env *TestEnvironment) LoadAndRefresh() { } else if env.needsRefresh { env.Config.Partner = configsections.TestPartner{} env.Config.TestTarget = configsections.TestTarget{} + env.TestOrchestrator = nil env.doAutodiscover() } } func (env *TestEnvironment) doAutodiscover() { + if len(env.Config.TargetNameSpaces) != 1 { + log.Fatal("a single namespace should be specified in config file") + } + env.NameSpaceUnderTest = env.Config.TargetNameSpaces[0].Name if autodiscover.PerformAutoDiscovery() { - autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget) + autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget, env.NameSpaceUnderTest) } - autodiscover.FindTestPartner(&env.Config.Partner) + autodiscover.FindTestPartner(&env.Config.Partner, env.NameSpaceUnderTest) log.Infof("Test Configuration: %+v", *env) @@ -182,7 +188,6 @@ func (env *TestEnvironment) doAutodiscover() { env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest - log.Info(env.TestOrchestrator) log.Info(env.ContainersUnderTest) env.needsRefresh = false diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index b9c740b34..2bd7d5816 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -40,10 +40,18 @@ type Operator struct { SubscriptionName string `yaml:"subscriptionName" json:"subscriptionName"` } +// Namespace struct defines namespace properties +type Namespace struct { + Name string `yaml:"name" json:"name"` +} + // TestConfiguration provides test related configuration type TestConfiguration struct { // Custom Pod labels for discovering containers/pods under test TargetPodLabels []Label `yaml:"targetPodLabels,omitempty" json:"targetPodLabels,omitempty"` + // targetNameSpaces to be used in + TargetNameSpaces []Namespace `yaml:"targetNameSpaces" json:"targetNameSpaces"` + // TestTarget contains k8s resources that can be targeted by tests TestTarget `yaml:"testTarget" json:"testTarget"` // TestPartner contains the helper containers that can be used to facilitate tests diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index b3da11e3c..88b029f43 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -73,6 +73,9 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestICMPv4ConnectivityIdentifier) ginkgo.It(testID, func() { + if env.TestOrchestrator == nil { + ginkgo.Skip("Orchestrator is not deployed, skip this test") + } for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue @@ -97,6 +100,9 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.When("Testing network connectivity", func() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestICMPv4ConnectivityIdentifier) ginkgo.It(testID, func() { + if env.TestOrchestrator == nil { + ginkgo.Skip("Orchestrator is not deployed, skip this test") + } for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 859506a1b..725a38cde 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -1,3 +1,5 @@ +targetNameSpaces: + - name: tnf targetPodLabels: - prefix: test-network-function.com name: generic From b463115b47bd38b4f85a1e5d39b66eb728f42638 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 9 Sep 2021 11:47:34 -0500 Subject: [PATCH 054/344] re-enable operator tests (#351) --- .github/workflows/pre-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index bae7a8e41..38ffbf1f4 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -125,7 +125,7 @@ jobs: run: ./run-cnf-suites.sh --focus diagnostic - name: 'Test: Run test suites' - run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification + run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification operator # Perform smoke tests using a TNF container. - name: Build the `test-network-function` image @@ -148,7 +148,7 @@ jobs: run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator # Push the new unstable TNF image to Quay.io. From dd7a54905c43ef27f895d491654b714841a6cad6 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 9 Sep 2021 12:05:59 -0500 Subject: [PATCH 055/344] Fix for operator tests passing with no operator (#349) * Fix for operator tests passing with no operator * skipping instead of failing Co-authored-by: Salaheddine Hamadi --- pkg/config/config.go | 2 ++ test-network-function/operator/suite.go | 28 ++++++++++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index c75b5a1b4..17670b8e8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -116,6 +116,7 @@ type TestEnvironment struct { PartnerContainers map[configsections.ContainerIdentifier]*Container PodsUnderTest []configsections.Pod DeploymentsUnderTest []configsections.Deployment + OperatorsUnderTest []configsections.Operator NameSpaceUnderTest string // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. @@ -188,6 +189,7 @@ func (env *TestEnvironment) doAutodiscover() { env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest + env.OperatorsUnderTest = env.Config.Operators log.Info(env.TestOrchestrator) log.Info(env.ContainersUnderTest) env.needsRefresh = false diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 2af9f3a7f..ced61b800 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -31,7 +31,6 @@ import ( ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/config" - "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/operator" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" @@ -70,6 +69,9 @@ var _ = ginkgo.Describe(testSpecName, func() { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() + if len(env.OperatorsUnderTest) == 0 { + ginkgo.Skip("No Operator found.") + } }) defer ginkgo.GinkgoRecover() @@ -82,18 +84,17 @@ var _ = ginkgo.Describe(testSpecName, func() { }) }) ginkgo.Context("Runs test on operators", func() { - itRunsTestsOnOperator() + itRunsTestsOnOperator(env) }) - testOperatorsAreInstalledViaOLM() + testOperatorsAreInstalledViaOLM(env) } }) // testOperatorsAreInstalledViaOLM ensures all configured operators have a proper OLM subscription. -func testOperatorsAreInstalledViaOLM() { +func testOperatorsAreInstalledViaOLM(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsInstalledViaOLMIdentifier) ginkgo.It(testID, func() { - _, operatorsInTest := getConfig() - for _, operatorInTest := range operatorsInTest { + for _, operatorInTest := range env.OperatorsUnderTest { defer results.RecordResult(identifiers.TestOperatorIsInstalledViaOLMIdentifier) ginkgo.By(fmt.Sprintf("%s in namespace %s Should have a valid subscription", operatorInTest.SubscriptionName, operatorInTest.Namespace)) testOperatorIsInstalledViaOLM(operatorInTest.SubscriptionName, operatorInTest.Namespace) @@ -123,14 +124,7 @@ func testOperatorIsInstalledViaOLM(subscriptionName, subscriptionNamespace strin gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) } -func getConfig() ([]configsections.CertifiedOperatorRequestInfo, []configsections.Operator) { - conf := config.GetTestEnvironment().Config - operatorsToQuery := conf.CertifiedOperatorInfo - operatorsInTest := conf.Operators - return operatorsToQuery, operatorsInTest -} - -func itRunsTestsOnOperator() { +func itRunsTestsOnOperator(env *config.TestEnvironment) { for _, testType := range testcases.GetConfiguredOperatorTests() { testFile, err := testcases.LoadConfiguredTestFile(configuredTestFile) gomega.Expect(testFile).ToNot(gomega.BeNil()) @@ -143,17 +137,17 @@ func itRunsTestsOnOperator() { if testCase.SkipTest { continue } - runTestsOnOperator(testCase) + runTestsOnOperator(env, testCase) } } } //nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. -func runTestsOnOperator(testCase testcases.BaseTestCase) { +func runTestsOnOperator(env *config.TestEnvironment, testCase testcases.BaseTestCase) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorInstallStatusIdentifier) + "-" + testCase.Name ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestOperatorInstallStatusIdentifier) - for _, op := range config.GetTestEnvironment().Config.Operators { + for _, op := range env.OperatorsUnderTest { if testCase.ExpectedType == testcases.Function { for _, val := range testCase.ExpectedStatus { testCase.ExpectedStatusFn(op.Name, testcases.StatusFunctionType(val)) From 97a66149a648ea2349595093f8929ba64f99cb58 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 10 Sep 2021 14:34:59 -0400 Subject: [PATCH 056/344] Ignore Error in FSDIFF (#352) --- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 9a7dd0b21..67b396337 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -41,7 +41,6 @@ const ( usrbin = `(?m)[\t|\s]\/usr\/bin[.]*` usrsbin = `(?m)[\t|\s]\/usr\/sbin[.]*` usrlib = `(?m)[\t|\s]\/usr\/lib[.]*` - err = `(?m)Error` successfulOutputRegex = `(?m){}` acceptAllRegex = `(?m)(.|\n)+` ) @@ -79,8 +78,6 @@ func (p *CnfFsDiff) ReelFirst() *reel.Step { func (p *CnfFsDiff) ReelMatch(pattern, before, match string) *reel.Step { p.result = tnf.SUCCESS switch pattern { - case err: - p.result = tnf.ERROR case varlibrpm, varlibdpkg, bin, sbin, lib, usrbin, usrsbin, usrlib: p.result = tnf.FAILURE case successfulOutputRegex: @@ -114,5 +111,5 @@ func NewFsDiff(timeout time.Duration, containerID, nodeName string) *CnfFsDiff { // GetReelFirstRegularExpressions returns the regular expressions used for matching in ReelFirst. func (p *CnfFsDiff) GetReelFirstRegularExpressions() []string { - return []string{err, varlibrpm, varlibdpkg, bin, sbin, lib, usrbin, usrsbin, usrlib, successfulOutputRegex, acceptAllRegex} + return []string{varlibrpm, varlibdpkg, bin, sbin, lib, usrbin, usrsbin, usrlib, successfulOutputRegex, acceptAllRegex} } From edb46bb240fa57bb1949a7132d6ddbb6f650a9be Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Tue, 14 Sep 2021 14:05:46 -0500 Subject: [PATCH 057/344] Matching latest TNF and Partner repo versions to build latest release image (#354) * Matching latest TNF and Partner repo versions to build latest release image * removing testing code --- .github/workflows/tnf-image.yaml | 30 +++++++++++++++++++++++------- version.json | 3 ++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 79886b41b..8e84c45ec 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -28,15 +28,18 @@ env: TNF_CONFIG_DIR: /tmp/tnf/config TNF_OUTPUT_DIR: /tmp/tnf/output TNF_SRC_URL: 'https://github.com/${{ github.repository }}' + PARTNER_REPO: test-network-function/cnf-certification-test-partner + PARTNER_SRC_URL: 'https://github.com/${PARTNER_REPO}' TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${IMAGE_NAME}:${IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}' jobs: - get-latest-tnf-version-number: - name: 'Get the version number of the latest release' + get-latest-release-version-numbers: + name: 'Get the version numbers of the latest release components' if: ${{ github.repository_owner == 'test-network-function' }} runs-on: ubuntu-20.04 outputs: TNF_VERSION: ${{ steps.set_tnf_version.outputs.version_number }} + PARTNER_VERSION: ${{ steps.set_tnf_version.outputs.partner_version_number }} steps: - name: Checkout generic working branch of the current version @@ -52,22 +55,26 @@ jobs: run: | echo Version tag: $VERSION_FROM_FILE echo "::set-output name=version_number::$VERSION_FROM_FILE" + echo Partner version tag: $VERSION_FROM_FILE_PARTNER + echo "::set-output name=partner_version_number::$VERSION_FROM_FILE_PARTNER" id: set_tnf_version env: - VERSION_FROM_FILE: ${{ fromJSON(steps.get_version_json_file.outputs.json)['tag'] }} + VERSION_FROM_FILE: ${{ fromJSON(steps.get_version_json_file.outputs.json).tag }} + VERSION_FROM_FILE_PARTNER: ${{ fromJSON(steps.get_version_json_file.outputs.json).partner_tag }} test-and-push-tnf-image: name: 'Test and push the `test-network-function` image' - needs: [get-latest-tnf-version-number] + needs: [get-latest-release-version-numbers] runs-on: ubuntu-20.04 env: SHELL: /bin/bash KUBECONFIG: '/home/runner/.kube/config' - TNF_VERSION: ${{ needs['get-latest-tnf-version-number'].outputs.TNF_VERSION }} + TNF_VERSION: ${{ needs['get-latest-release-version-numbers'].outputs.TNF_VERSION }} + PARTNER_VERSION: ${{ needs['get-latest-release-version-numbers'].outputs.PARTNER_VERSION }} steps: - name: Ensure $TNF_VERSION and $IMAGE_TAG are set - run: '[[ -n "$TNF_VERSION" ]] && [[ -n "$IMAGE_TAG" ]]' + run: '[[ -n "$TNF_VERSION" ]] && [[ -n "$IMAGE_TAG" ]] && [[ -n "$PARTNER_VERSION" ]]' - name: Check whether the version tag exists on remote run: git ls-remote --exit-code $TNF_SRC_URL refs/tags/$TNF_VERSION @@ -76,6 +83,13 @@ jobs: if: ${{ failure() }} run: echo "Tag '$TNF_VERSION' does not exist on remote $TNF_SRC_URL" + - name: Check whether the version tag exists on remote + run: git ls-remote --exit-code ${{ env.PARTNER_SRC_URL }} refs/tags/$PARTNER_VERSION + + - name: (if partner_tag is missing) Display debug message + if: ${{ failure() }} + run: echo "Tag '$PARTNER_VERSION' does not exist on remote $PARTNER_SRC_URL" + - name: Checkout the version tag uses: actions/checkout@v2 with: @@ -97,7 +111,9 @@ jobs: with: repository: test-network-function/cnf-certification-test-partner path: cnf-certification-test-partner - + with: + ref: ${{ env.PARTNER_VERSION }} + - name: Start the minikube cluster for `local-test-infra` uses: ./cnf-certification-test-partner/.github/actions/start-minikube with: diff --git a/version.json b/version.json index cb475e476..086979156 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,4 @@ { - "tag": "v3.0.0" + "tag": "v3.0.0", + "partner_tag": "v3.0.0" } From a5e500c9dea4b626c5f07cd3fbc8831515ae3c35 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 15 Sep 2021 17:42:53 -0400 Subject: [PATCH 058/344] clean up logging initialization (#355) * clean up logging initialization * Address PR comments * fix lint * fix lint * fix lint * fix lint * fix lint * fix lint --- go.sum | 13 ++-- pkg/tnf/reel/reel.go | 2 +- test-network-function/common/env.go | 64 ++++---------------- test-network-function/observability/suite.go | 1 + test-network-function/platform/suite.go | 25 ++------ 5 files changed, 26 insertions(+), 79 deletions(-) diff --git a/go.sum b/go.sum index ea9b0a506..2192a0995 100644 --- a/go.sum +++ b/go.sum @@ -50,10 +50,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -90,7 +88,6 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -102,6 +99,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= @@ -139,9 +137,13 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 h1:/nu0LtOLsZMlqfk6i50C5fmo5cp5WfciWggZYlwGNAg= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= +github.com/google/goexpect v0.0.0-20210914193350-0010e15e87b9 h1:HOFqDJo8gSnYucPGdobe/LylQLOmW9XpZ3sML1YBtg8= +github.com/google/goexpect v0.0.0-20210914193350-0010e15e87b9/go.mod h1:3bkiyiIBwCvUXYv811zuwnHlrPotynhld5LviGDXPfw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= +github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4= +github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -195,10 +197,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -446,6 +446,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -624,7 +626,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/tnf/reel/reel.go b/pkg/tnf/reel/reel.go index 7d1e01066..42731033a 100644 --- a/pkg/tnf/reel/reel.go +++ b/pkg/tnf/reel/reel.go @@ -106,7 +106,7 @@ func DisableTerminalPromptEmulation() Option { } // Each Step can have zero or more expectations (Step.Expect). This method follows the Adapter design pattern; a raw -// array of strings is turned into a corresponding array of exepct.Batcher. This method side-effects the input +// array of strings is turned into a corresponding array of expect.Batcher. This method side-effects the input // expectations array, following the Builder design pattern. Finally, the first match is stored in the firstMatch // output parameter. func (r *Reel) batchExpectations(expectations []string, batcher []expect.Batcher, firstMatch *string) []expect.Batcher { diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index d9fbc220d..afa1c307e 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -17,15 +17,13 @@ package common import ( - "errors" "os" "path" "strconv" - "strings" "time" "github.com/onsi/gomega" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" ) @@ -41,20 +39,6 @@ var ( schemaPath = path.Join("schemas", "generic-test.schema.json") ) -const ( - logLevelTraceString = "trace" - logLevelDebugString = "debug" - logLevelInfoString = "info" - logLevelWarnString = "warn" - logLevelErrorString = "error" - logLevelFatalString = "fatal" - logLevelPanicString = "panic" - logLevelEmptyString = "" - logLevelDefault = logrus.InfoLevel - errorEmpty = "empty" - errorInvalid = "invalid" -) - // DefaultTimeout for creating new interactive sessions (oc, ssh, tty) var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second @@ -91,47 +75,23 @@ func GetOcDebugImageID() string { return os.Getenv("TNF_OC_DEBUG_IMAGE_ID") } -// logLevel retrieves the LOG_LEVEL environement vaiable +// logLevel retrieves the LOG_LEVEL environment variable func logLevel() string { - return os.Getenv("LOG_LEVEL") -} - -// stringToLogLevel converts a string to a log logrus level -func stringToLogLevel(logLevelString string) (logrus.Level, error) { - logLevelString = strings.ToLower(logLevelString) - switch logLevelString { - case logLevelTraceString: - return logrus.TraceLevel, nil - case logLevelDebugString: - return logrus.DebugLevel, nil - case logLevelInfoString: - return logrus.InfoLevel, nil - case logLevelWarnString: - return logrus.WarnLevel, nil - case logLevelErrorString: - return logrus.ErrorLevel, nil - case logLevelFatalString: - return logrus.FatalLevel, nil - case logLevelPanicString: - return logrus.PanicLevel, nil - case logLevelEmptyString: - return logLevelDefault, errors.New(errorEmpty) + logLevel := os.Getenv("LOG_LEVEL") + if logLevel == "" { + log.Info("LOG_LEVEL environment is not set, defaulting to DEBUG") + logLevel = "debug" //nolint:goconst } - return logLevelDefault, errors.New(errorInvalid) + return logLevel } // SetLogLevel sets the log level for logrus based on the "LOG_LEVEL" environment variable func SetLogLevel() { - var aLogLevel, err = stringToLogLevel(logLevel()) - + var aLogLevel, err = log.ParseLevel(logLevel()) if err != nil { - if err.Error() == errorInvalid { - logrus.Info("LOG_LEVEL environment set with an invalid value, defaulting to", logLevelDefault, ".\n Valid values are: trace, debug, info, warn, error, fatal, panic") - } - if err.Error() == errorEmpty { - logrus.Info("LOG_LEVEL environment variable not set, defaulting to: ", logLevelDefault, ".\n Valid values are: trace, debug, info, warn, error, fatal, panic") - } + log.Error("LOG_LEVEL environment set with an invalid value, defaulting to DEBUG \n Valid values are: trace, debug, info, warn, error, fatal, panic") + aLogLevel = log.DebugLevel } - logrus.Info("Log level set to:", aLogLevel) - logrus.SetLevel(aLogLevel) + log.Info("Log level set to:", aLogLevel) + log.SetLevel(aLogLevel) } diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index bc4349036..c10f618e1 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -68,6 +68,7 @@ func testLogging(env *config.TestEnvironment) { } func loggingTest(c configsections.ContainerIdentifier) { context := common.GetContext() + values := make(map[string]interface{}) values["POD_NAMESPACE"] = c.Namespace values["POD_NAME"] = c.PodName diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 649694fef..6ac39171d 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -66,22 +66,13 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { // use this boolean to turn off tests that require OS packages if !common.IsMinikube() { testContainersFsDiff(env) + testTainted() + testHugepages() + testBootParams(env) + testSysctlConfigs(env) } + testIsRedHatRelease(env) }) - - testTainted() - testHugepages() - - if !common.IsMinikube() { - testBootParams(env) - } - - if !common.IsMinikube() { - testSysctlConfigs(env) - } - - testIsRedHatRelease(env) - } }) @@ -297,9 +288,6 @@ func testSysctlConfigsHelper(podName, podNamespace string) { } func testTainted() { - if common.IsMinikube() { - return - } var nodeNames []string testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) ginkgo.It(testID, func() { @@ -337,9 +325,6 @@ func testTainted() { } func testHugepages() { - if common.IsMinikube() { - return - } var nodeNames []string var clusterHugepages, clusterHugepagesz int testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) From 29611239da529de63c31ff61bc2666b9f7267877 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 16 Sep 2021 11:39:20 -0400 Subject: [PATCH 059/344] Oc session error handling (#357) --- pkg/config/config.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 17670b8e8..69c5f2e90 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,7 +74,6 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio // correctly. var containerOc *interactive.Oc ocChan := make(chan *interactive.Oc) - var chOut <-chan error goExpectSpawner := interactive.NewGoExpectSpawner() var spawner interactive.Spawner = goExpectSpawner @@ -83,15 +82,14 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio oc, outCh, err := interactive.SpawnOc(&spawner, pod, container, namespace, timeout, options...) gomega.Expect(outCh).ToNot(gomega.BeNil()) gomega.Expect(err).To(gomega.BeNil()) + // Set up a go routine which reads from the error channel + go func() { + err := <-outCh + gomega.Expect(err).To(gomega.BeNil()) + }() ocChan <- oc }() - // Set up a go routine which reads from the error channel - go func() { - err := <-chOut - gomega.Expect(err).To(gomega.BeNil()) - }() - containerOc = <-ocChan gomega.Expect(containerOc).ToNot(gomega.BeNil()) From ede89954918214d0883c3518d4b93d44545957fd Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 16 Sep 2021 14:10:04 -0400 Subject: [PATCH 060/344] add default timeout to send command in spawned shell (#356) add default timeout to send command in spawned shell This aims to add timeout to "Send" function in goexpect library, by default, the library does not use any timeout. By looking in the code of the goexpect, it's printing the log after "sending" the command, so if there's any issue in the "send" we will not see the log. This seems like the issue we were seeing this morning where the process hangs. --- cmd/generic/cmd/jsontest.go | 4 ++-- cmd/ssh/main.go | 2 +- pkg/config/config.go | 2 +- pkg/tnf/interactive/spawner.go | 19 +++++++++++++++++++ test-network-function/common/env.go | 2 +- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmd/generic/cmd/jsontest.go b/cmd/generic/cmd/jsontest.go index f9728960e..b8f335f53 100644 --- a/cmd/generic/cmd/jsontest.go +++ b/cmd/generic/cmd/jsontest.go @@ -224,7 +224,7 @@ func runSSHCmd(_ *cobra.Command, args []string) { // SSH shell creation. goExpectSpawner := interactive.NewGoExpectSpawner() var spawnContext interactive.Spawner = goExpectSpawner - context, err := interactive.SpawnSSH(&spawnContext, user, host, (*tester).Timeout(), interactive.Verbose(true)) + context, err := interactive.SpawnSSH(&spawnContext, user, host, (*tester).Timeout(), interactive.Verbose(true), interactive.SendTimeout((*tester).Timeout())) if err != nil { fatalError("could not create the ssh expecter", err, testExpecterCreationFailedExitCode) } @@ -251,7 +251,7 @@ func runOcCmd(_ *cobra.Command, args []string) { // oc shell creation. goExpectSpawner := interactive.NewGoExpectSpawner() var spawnContext interactive.Spawner = goExpectSpawner - oc, ch, err := interactive.SpawnOc(&spawnContext, pod, container, namespace, (*tester).Timeout(), interactive.Verbose(true)) + oc, ch, err := interactive.SpawnOc(&spawnContext, pod, container, namespace, (*tester).Timeout(), interactive.Verbose(true), interactive.SendTimeout((*tester).Timeout())) if err != nil { fatalError("could not create the oc expecter", err, testExpecterCreationFailedExitCode) } diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index 23f57663b..c9be31e2e 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -52,7 +52,7 @@ func parseArgs() (*interactive.Context, string, time.Duration, error) { timeoutDuration := time.Duration(*timeout) * time.Second goExpectSpawner := interactive.NewGoExpectSpawner() var spawner interactive.Spawner = goExpectSpawner - context, err := interactive.SpawnSSH(&spawner, args[0], args[1], timeoutDuration, interactive.Verbose(true)) + context, err := interactive.SpawnSSH(&spawner, args[0], args[1], timeoutDuration, interactive.Verbose(true), interactive.SendTimeout(timeoutDuration)) return context, args[2], timeoutDuration, err } diff --git a/pkg/config/config.go b/pkg/config/config.go index 69c5f2e90..9732a180d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -198,7 +198,7 @@ func (env *TestEnvironment) doAutodiscover() { func (env *TestEnvironment) createContainers(containerDefinitions []configsections.ContainerConfig) map[configsections.ContainerIdentifier]*Container { createdContainers := make(map[configsections.ContainerIdentifier]*Container) for _, c := range containerDefinitions { - oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true)) + oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true), interactive.SendTimeout(DefaultTimeout)) var defaultIPAddress = "UNKNOWN" if _, ok := env.ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier]; !ok { defaultIPAddress = getContainerDefaultNetworkIPAddress(oc, c.DefaultNetworkDevice) diff --git a/pkg/tnf/interactive/spawner.go b/pkg/tnf/interactive/spawner.go index 21e1ca783..3256279d4 100644 --- a/pkg/tnf/interactive/spawner.go +++ b/pkg/tnf/interactive/spawner.go @@ -146,6 +146,11 @@ type GoExpectSpawner struct { verboseWriterIsSet bool // verboseWriter is an alternate destination for verbose logs. verboseWriter io.Writer + + // sendTimeoutIsSet tracks whether the Send command timeout is set. + sendTimeoutIsSet bool + // sendTimeout is the timeout of send command + sendTimeout time.Duration } // Option is a function pointer to enable lightweight optionals for GoExpectSpawner. @@ -191,6 +196,16 @@ func VerboseWriter(verboseWriter io.Writer) Option { } } +// SendTimeout sets the timeout of send command +func SendTimeout(timeout time.Duration) Option { + return func(g *GoExpectSpawner) Option { + g.sendTimeoutIsSet = true + prev := g.sendTimeout + g.sendTimeout = timeout + return SendTimeout(prev) + } +} + // getDefaultBufferSize returns the default buffer size as sourced from TNF_DEFAULT_BUFFER_SIZE. If // TNF_DEFAULT_BUFFER_SIZE is not set or cannot be parsed as an integer, defaultBufferSize is returned. func getDefaultBufferSize() int { @@ -228,6 +243,10 @@ func (g *GoExpectSpawner) GetGoExpectOptions() []expect.Option { opts = append(opts, expect.VerboseWriter(g.verboseWriter)) } + if g.sendTimeoutIsSet { + opts = append(opts, expect.SendTimeout(g.sendTimeout)) + } + return opts } diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index afa1c307e..ed30eae40 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -44,7 +44,7 @@ var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second // GetContext spawns a new shell session and returns its context func GetContext() *interactive.Context { - context, err := interactive.SpawnShell(interactive.CreateGoExpectSpawner(), DefaultTimeout, interactive.Verbose(true)) + context, err := interactive.SpawnShell(interactive.CreateGoExpectSpawner(), DefaultTimeout, interactive.Verbose(true), interactive.SendTimeout(DefaultTimeout)) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(context).ToNot(gomega.BeNil()) gomega.Expect(context.GetExpecter()).ToNot(gomega.BeNil()) From 63808c01bf8e8039fe7ff120c17496899039e27c Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 16 Sep 2021 18:37:30 -0400 Subject: [PATCH 061/344] disable optimization and inlining in default build (#358) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef980a4b1..58abc9e33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ WORKDIR ${TNF_SRC_DIR} RUN make install-tools && \ make mocks && \ make update-deps && \ - make build-cnf-tests + make build-cnf-tests-debug # Extract what's needed to run at a seperate location RUN mkdir ${TNF_BIN_DIR} && \ From 77472134396521d2fcd2c104aec62e68e5c4b3bf Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 21 Sep 2021 09:22:15 -0400 Subject: [PATCH 062/344] Make network address discovery optional (#359) --- pkg/config/config.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 9732a180d..2ae19ac7d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -99,13 +99,17 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio // Extract a container IP address for a particular device. This is needed since container default network IP address // is served by dhcp, and thus is ephemeral. -func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) string { +func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) (string, error) { log.Infof("Getting IP Information for: %s(%s) in ns=%s", oc.GetPodName(), oc.GetPodContainerName(), oc.GetPodNamespace()) ipTester := ipaddr.NewIPAddr(DefaultTimeout, dev) test, err := tnf.NewTest(oc.GetExpecter(), ipTester, []reel.Handler{ipTester}, oc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidateTest() - return ipTester.GetIPv4Address() + result, err := test.Run() + if result == tnf.SUCCESS && err == nil { + return ipTester.GetIPv4Address(), nil + } + return "", fmt.Errorf("failed to get IP information for %s(%s) in ns=%s, result=%v, err=%v", + oc.GetPodName(), oc.GetPodContainerName(), oc.GetPodNamespace(), result, err) } // TestEnvironment includes the representation of the current state of the test targets and parters as well as the test configuration @@ -200,8 +204,13 @@ func (env *TestEnvironment) createContainers(containerDefinitions []configsectio for _, c := range containerDefinitions { oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true), interactive.SendTimeout(DefaultTimeout)) var defaultIPAddress = "UNKNOWN" + var err error if _, ok := env.ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier]; !ok { - defaultIPAddress = getContainerDefaultNetworkIPAddress(oc, c.DefaultNetworkDevice) + defaultIPAddress, err = getContainerDefaultNetworkIPAddress(oc, c.DefaultNetworkDevice) + if err != nil { + log.Warnf("Adding container to the ExcludeFromConnectivityTests list due to: %v", err) + env.ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier] = "" + } } createdContainers[c.ContainerIdentifier] = &Container{ ContainerConfiguration: c, From dc2b3cd147b9a22e8e2097ea2e12d2a098e4d95d Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Tue, 21 Sep 2021 20:02:04 +0200 Subject: [PATCH 063/344] Spawner Check function implementation. (#361) Calls to (*expecter).Send() will fail if the shell process has already exited. --- pkg/tnf/interactive/spawner.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pkg/tnf/interactive/spawner.go b/pkg/tnf/interactive/spawner.go index 3256279d4..4c29c8c45 100644 --- a/pkg/tnf/interactive/spawner.go +++ b/pkg/tnf/interactive/spawner.go @@ -21,6 +21,7 @@ import ( "os" "os/exec" "strconv" + "strings" "time" expect "github.com/google/goexpect" @@ -60,6 +61,12 @@ type SpawnFunc interface { // Wait consult exec.Cmd.Wait Wait() error + + // IsRunning returns true if the shell hasn't exited yet. + IsRunning() bool + + // Args returns the command and arguments used to spawn the shell. + Args() []string } // ExecSpawnFunc is an implementation of SpawnFunc using exec.Cmd. @@ -80,6 +87,16 @@ func (e *ExecSpawnFunc) Wait() error { return e.cmd.Wait() } +// IsRunning returns true if e.Cmd.ProcessState is nil, false otherwise +func (e *ExecSpawnFunc) IsRunning() bool { + return e.cmd.ProcessState == nil +} + +// Args wraps e.Cmd.Args +func (e *ExecSpawnFunc) Args() []string { + return e.cmd.Args +} + // Start wraps exec.Cmd.Start. func (e *ExecSpawnFunc) Start() error { return e.cmd.Start() @@ -296,7 +313,13 @@ func (g *GoExpectSpawner) spawnGeneric(spawnFunc *SpawnFunc, stdinPipe io.WriteC Close: func() error { return nil }, - Check: func() bool { return true }, + Check: func() bool { + if !(*spawnFunc).IsRunning() { + log.Error("Unable to spawn shell. Cmd: " + strings.Join((*spawnFunc).Args(), " ")) + return false + } + return true + }, }, timeout, opts...) // coax out the typing var expecter expect.Expecter = gexpecter From 162719789e9d60dea924e64f6594fe69849a0a76 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 22 Sep 2021 09:44:48 -0400 Subject: [PATCH 064/344] Quick fix for platform suite test context (#360) * Quick fix for platform suite test context * Update suite.go Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- test-network-function/platform/suite.go | 52 ++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 6ac39171d..f8b3390d7 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -62,17 +62,15 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) - ginkgo.Context("Container does not have additional packages installed", func() { - // use this boolean to turn off tests that require OS packages - if !common.IsMinikube() { - testContainersFsDiff(env) - testTainted() - testHugepages() - testBootParams(env) - testSysctlConfigs(env) - } - testIsRedHatRelease(env) - }) + // use this boolean to turn off tests that require OS packages + if !common.IsMinikube() { + testContainersFsDiff(env) + testTainted() + testHugepages() + testBootParams(env) + testSysctlConfigs(env) + } + testIsRedHatRelease(env) } }) @@ -104,22 +102,24 @@ func testContainerIsRedHatRelease(cut *config.Container) { // testContainersFsDiff test that all CUT didn't install new packages are starting func testContainersFsDiff(env *config.TestEnvironment) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredBaseImageIdentifier) - ginkgo.It(testID, func() { - var badContainers []string - for _, cut := range env.ContainersUnderTest { - podName := cut.Oc.GetPodName() - containerName := cut.Oc.GetPodContainerName() - context := cut.Oc - nodeName := cut.ContainerConfiguration.NodeName - ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) - testResult, err := testContainerFsDiff(nodeName, context) - if testResult != tnf.SUCCESS || err != nil { - badContainers = append(badContainers, containerName) - ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) + ginkgo.Context("Container does not have additional packages installed", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredBaseImageIdentifier) + ginkgo.It(testID, func() { + var badContainers []string + for _, cut := range env.ContainersUnderTest { + podName := cut.Oc.GetPodName() + containerName := cut.Oc.GetPodContainerName() + context := cut.Oc + nodeName := cut.ContainerConfiguration.NodeName + ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) + testResult, err := testContainerFsDiff(nodeName, context) + if testResult != tnf.SUCCESS || err != nil { + badContainers = append(badContainers, containerName) + ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) + } } - } - gomega.Expect(badContainers).To(gomega.BeNil()) + gomega.Expect(badContainers).To(gomega.BeNil()) + }) }) } From 58303403a4c73cc990b099dae44b18ecaed50dff Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:36:56 -0500 Subject: [PATCH 065/344] Adding stderr to logger (#362) * Adding stderr to logger * Adding stdout logger as well * PR review * Fix multilines missing --- pkg/tnf/interactive/spawner.go | 65 ++++++++++++++++++++++++++--- pkg/tnf/interactive/spawner_test.go | 58 ++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/pkg/tnf/interactive/spawner.go b/pkg/tnf/interactive/spawner.go index 4c29c8c45..b810d6175 100644 --- a/pkg/tnf/interactive/spawner.go +++ b/pkg/tnf/interactive/spawner.go @@ -17,6 +17,8 @@ package interactive import ( + "bufio" + "fmt" "io" "os" "os/exec" @@ -59,6 +61,9 @@ type SpawnFunc interface { // StdoutPipe consult exec.Cmd.StdoutPipe StdoutPipe() (io.Reader, error) + // StderrPipe consult exec.Cmd.StderrPipe + StderrPipe() (io.Reader, error) + // Wait consult exec.Cmd.Wait Wait() error @@ -112,6 +117,11 @@ func (e *ExecSpawnFunc) StdoutPipe() (io.Reader, error) { return e.cmd.StdoutPipe() } +// StderrPipe wraps exec.Cmd.Stderrpipe +func (e *ExecSpawnFunc) StderrPipe() (io.Reader, error) { + return e.cmd.StderrPipe() +} + // Spawner provides an interface for creating interactive sessions such as oc, ssh, or shell. type Spawner interface { // Spawn creates the interactive session. @@ -272,6 +282,31 @@ func NewGoExpectSpawner() *GoExpectSpawner { return &GoExpectSpawner{} } +// logCmdMirrorPipe logs specified pipe output to logger. +func logCmdMirrorPipe(cmdLine string, pipeToMirror io.Reader, name string, trace bool) io.Reader { + originalPipe := pipeToMirror + r, w, _ := os.Pipe() + tr := io.TeeReader(originalPipe, w) + + go func() { + buf := bufio.NewReader(tr) + for { + line, _, err := buf.ReadLine() + if trace { + log.Trace(name + " for " + cmdLine + " : " + string(line)) + } else { + log.Warn(name + " for " + cmdLine + " : " + string(line)) + } + if err != nil { + // Some Error has happened, goroutine about to exit + log.Warnf("Exiting cmd log mirroring goroutine. Error: %s", err) + return + } + } + }() + return r +} + // Spawn creates a subprocess, setting standard input and standard output appropriately. This is the base method to // create any interactive PTY based process. func (g *GoExpectSpawner) Spawn(command string, args []string, timeout time.Duration, opts ...Option) (*Context, error) { @@ -286,10 +321,15 @@ func (g *GoExpectSpawner) Spawn(command string, args []string, timeout time.Dura } spawnFunc = (*spawnFunc).Command(command, args...) - stdinPipe, stdoutPipe, err := g.unpackPipes(spawnFunc) + stdinPipe, stdoutPipe, stderrPipe, err := g.unpackPipes(spawnFunc) if err != nil { return nil, err } + + cmdLine := fmt.Sprintf("%s %s", command, strings.Join(args, " ")) + logCmdMirrorPipe(cmdLine, stderrPipe, "STDERR", false) + stdoutPipe = logCmdMirrorPipe(cmdLine, stdoutPipe, "STDOUT", true) + err = g.startCommand(spawnFunc, command, args) if err != nil { return nil, err @@ -337,17 +377,21 @@ func (g *GoExpectSpawner) startCommand(spawnFunc *SpawnFunc, command string, arg return err } -// Helper method to unpack stdin and stdout. -func (g *GoExpectSpawner) unpackPipes(spawnFunc *SpawnFunc) (io.WriteCloser, io.Reader, error) { +//nolint:gocritic // Helper method to unpack stdin and stdout. +func (g *GoExpectSpawner) unpackPipes(spawnFunc *SpawnFunc) (io.WriteCloser, io.Reader, io.Reader, error) { stdinPipe, err := g.extractStdinPipe(spawnFunc) if err != nil { - return nil, nil, err + return nil, nil, nil, err } stdoutPipe, err := g.extractStdoutPipe(spawnFunc) if err != nil { - return nil, nil, err + return nil, nil, nil, err + } + stderrPipe, err := g.extractStderrPipe(spawnFunc) + if err != nil { + return nil, nil, nil, err } - return stdinPipe, stdoutPipe, err + return stdinPipe, stdoutPipe, stderrPipe, err } // Helper method to extract stdin. @@ -368,6 +412,15 @@ func (g *GoExpectSpawner) extractStdoutPipe(spawnFunc *SpawnFunc) (io.Reader, er return stdout, err } +// Helper method to extract stdout. +func (g *GoExpectSpawner) extractStderrPipe(spawnFunc *SpawnFunc) (io.Reader, error) { + stderr, err := (*spawnFunc).StderrPipe() + if err != nil { + log.Errorf("Couldn't extract stderr for the given process: %v", err) + } + return stderr, err +} + // CreateGoExpectSpawner creates a GoExpectSpawner implementation and returns it as a *Spawner for type compatibility // reasons. func CreateGoExpectSpawner() *Spawner { diff --git a/pkg/tnf/interactive/spawner_test.go b/pkg/tnf/interactive/spawner_test.go index c489ece5e..5dae02880 100644 --- a/pkg/tnf/interactive/spawner_test.go +++ b/pkg/tnf/interactive/spawner_test.go @@ -44,6 +44,7 @@ func init() { var ( defaultGoExpectArgs = []interactive.Option{interactive.Verbose(true)} defaultStdout, defaultStdin, _ = os.Pipe() + defaultStderr, _, _ = os.Pipe() errStart = errors.New("start failed") errStdInPipe = errors.New("failed to access stdin") ) @@ -62,6 +63,10 @@ type goExpectSpawnerTestCase struct { stdoutPipeReturnValue io.Reader stdoutPipeReturnErr error + stderrPipeShouldBeCalled bool + stderrPipeReturnValue io.Reader + stderrPipeReturnErr error + startShouldBeCalled bool startReturnErr error @@ -87,6 +92,10 @@ var goExpectSpawnerTestCases = map[string]goExpectSpawnerTestCase{ stdoutPipeReturnValue: nil, stdoutPipeReturnErr: nil, + stderrPipeShouldBeCalled: false, + stderrPipeReturnValue: nil, + stderrPipeReturnErr: nil, + startShouldBeCalled: false, startReturnErr: nil, @@ -110,13 +119,44 @@ var goExpectSpawnerTestCases = map[string]goExpectSpawnerTestCase{ stdoutPipeReturnValue: nil, stdoutPipeReturnErr: errStdInPipe, + stderrPipeShouldBeCalled: false, + stderrPipeReturnValue: nil, + stderrPipeReturnErr: nil, + + startShouldBeCalled: false, + startReturnErr: nil, + + goExpectSpawnerSpawnReturnContextIsNil: true, + goExpectSpawnerSpawnReturnErr: errStdInPipe, + }, + // 2. Progressing past the creation of stdin and stdout, now cause stderr to fail. + "stderr_pipe_creation_failure": { + // The command is unimportant + goExpectSpawnerSpawnCommand: "ls", + goExpectSpawnerSpawnArgs: []string{"-al"}, + goExpectSpawnerSpawnTimeout: testTimeoutDuration, + goExpectSpawnerSpawnOpts: defaultGoExpectArgs, + + stdinPipeShouldBeCalled: true, + stdinPipeReturnValue: defaultStdin, + stdinPipeReturnErr: nil, + + stdoutPipeShouldBeCalled: true, + stdoutPipeReturnValue: defaultStdout, + stdoutPipeReturnErr: nil, + + // cause StderrPipe() call to fail and ensure the error cascades. + stderrPipeShouldBeCalled: true, + stderrPipeReturnValue: nil, + stderrPipeReturnErr: errStdInPipe, + startShouldBeCalled: false, startReturnErr: nil, goExpectSpawnerSpawnReturnContextIsNil: true, goExpectSpawnerSpawnReturnErr: errStdInPipe, }, - // 3. Progressing past the creation of stdin/stdout, now cause Start to fail. + // 3. Progressing past the creation of stdin/stdout/stderr, now cause Start to fail. "start_failure": { // The command is unimportant goExpectSpawnerSpawnCommand: "ls", @@ -132,6 +172,10 @@ var goExpectSpawnerTestCases = map[string]goExpectSpawnerTestCase{ stdoutPipeReturnValue: defaultStdout, stdoutPipeReturnErr: nil, + stderrPipeShouldBeCalled: true, + stderrPipeReturnValue: defaultStderr, + stderrPipeReturnErr: nil, + // cause Start() call to fail and make sure the error cascades out of Spawn(). startShouldBeCalled: true, startReturnErr: errStart, @@ -155,6 +199,10 @@ var goExpectSpawnerTestCases = map[string]goExpectSpawnerTestCase{ stdoutPipeReturnValue: defaultStdout, stdoutPipeReturnErr: nil, + stderrPipeShouldBeCalled: true, + stderrPipeReturnValue: defaultStderr, + stderrPipeReturnErr: nil, + startShouldBeCalled: true, startReturnErr: nil, @@ -181,6 +229,10 @@ func TestGoExpectSpawner_Spawn(t *testing.T) { mockSpawnFunc.EXPECT().StdoutPipe().Return(testCase.stdoutPipeReturnValue, testCase.stdoutPipeReturnErr) } + if testCase.stderrPipeShouldBeCalled { + mockSpawnFunc.EXPECT().StderrPipe().Return(testCase.stderrPipeReturnValue, testCase.stderrPipeReturnErr) + } + if testCase.startShouldBeCalled { mockSpawnFunc.EXPECT().Start().Return(testCase.startReturnErr) } @@ -225,6 +277,10 @@ func TestExecSpawnFunc(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, stdout) + stderr, err := (*cmd).StderrPipe() + assert.Nil(t, err) + assert.NotNil(t, stderr) + err = (*cmd).Start() assert.Nil(t, err) From edb3df4c13b10bb6e71f0d52c8b752161c59319c Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Thu, 23 Sep 2021 11:24:52 -0400 Subject: [PATCH 066/344] add timestamp to log function (#364) add filename and line to log function --- test-network-function/common/env.go | 16 ++++++++++++++++ test-network-function/suite_test.go | 1 + 2 files changed, 17 insertions(+) diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index ed30eae40..52ede320e 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -17,8 +17,10 @@ package common import ( + "fmt" "os" "path" + "runtime" "strconv" "time" @@ -95,3 +97,17 @@ func SetLogLevel() { log.Info("Log level set to:", aLogLevel) log.SetLevel(aLogLevel) } + +// SetLogFormat sets the log format for logrus +func SetLogFormat() { + customFormatter := new(log.TextFormatter) + customFormatter.TimestampFormat = time.StampMilli + customFormatter.PadLevelText = true + customFormatter.FullTimestamp = true + log.SetReportCaller(true) + customFormatter.CallerPrettyfier = func(f *runtime.Frame) (string, string) { + _, filename := path.Split(f.File) + return strconv.Itoa(f.Line) + "]", fmt.Sprintf("[%s:", filename) + } + log.SetFormatter(customFormatter) +} diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 738e52c84..750b28b7c 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -116,6 +116,7 @@ func TestTest(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) log.Info("Version: ", programVersion, " ( ", GitCommit, " )") common.SetLogLevel() + common.SetLogFormat() tnfcommon.OcDebugImageID = common.GetOcDebugImageID() // Initialize the claim with the start time, tnf version, etc. From 9675ade9dc1a8d001225ac37b1029ef57fce7904 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 23 Sep 2021 13:22:55 -0400 Subject: [PATCH 067/344] Oc session error channel monitoring enhancement for intrusive test (#363) * Oc session error channel monitoring enhancement for intrusive test * Remove redundant assertion * Replace assertion with exit and improve the log text --- pkg/config/config.go | 11 +++- pkg/tnf/interactive/oc.go | 14 +++++- test-network-function/lifecycle/suite.go | 64 ++++++++++++------------ 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2ae19ac7d..5d1ac440a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -84,8 +84,15 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio gomega.Expect(err).To(gomega.BeNil()) // Set up a go routine which reads from the error channel go func() { - err := <-outCh - gomega.Expect(err).To(gomega.BeNil()) + for { + select { + case err := <-outCh: + log.Fatalf("OC session to container %s/%s is broken due to: %v, aborting the test run", oc.GetPodName(), oc.GetPodContainerName(), err) + os.Exit(1) + case <-oc.GetDoneChannel(): + break + } + } }() ocChan <- oc }() diff --git a/pkg/tnf/interactive/oc.go b/pkg/tnf/interactive/oc.go index d0e95fff4..183b0690b 100644 --- a/pkg/tnf/interactive/oc.go +++ b/pkg/tnf/interactive/oc.go @@ -52,6 +52,8 @@ type Oc struct { spawnErr error // error channel for interactive error stream errorChannel <-chan error + // done channel to notify the go routine that monitors the error channel + doneChannel chan bool } // SpawnOc creates an OpenShift Client subprocess, spawning the appropriate underlying PTY. @@ -62,7 +64,7 @@ func SpawnOc(spawner *Spawner, pod, container, namespace string, timeout time.Du return nil, context.GetErrorChannel(), err } errorChannel := context.GetErrorChannel() - return &Oc{pod: pod, container: container, namespace: namespace, timeout: timeout, opts: opts, expecter: context.GetExpecter(), spawnErr: err, errorChannel: errorChannel}, errorChannel, nil + return &Oc{pod: pod, container: container, namespace: namespace, timeout: timeout, opts: opts, expecter: context.GetExpecter(), spawnErr: err, errorChannel: errorChannel, doneChannel: make(chan bool)}, errorChannel, nil } // GetExpecter returns a reference to the expect.Expecter reference used to control the OpenShift client. @@ -109,3 +111,13 @@ func (o *Oc) GetOptions() []Option { func (o *Oc) GetErrorChannel() <-chan error { return o.errorChannel } + +// GetDoneChannel returns the receive only done channel +func (o *Oc) GetDoneChannel() <-chan bool { + return o.doneChannel +} + +// Close sends the signal to the done channel +func (o *Oc) Close() { + o.doneChannel <- true +} diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 012696551..1da3e3ed3 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -275,54 +275,54 @@ func shutdownTest(podNamespace, podName string) { func testPodsRecreation(env *config.TestEnvironment) { var deployments dp.DeploymentMap var notReadyDeployments []string - nodesNames := make(map[string]node) - namespaces := make(map[string]bool) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRecreationIdentifier) ginkgo.It(testID, func() { env.SetNeedsRefresh() ginkgo.By("Testing node draining effect of deployment") - for _, podUnderTest := range env.PodsUnderTest { - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("test deployment in namespace %s", podNamespace)) - deployments, notReadyDeployments = getDeployments(podNamespace) - if len(deployments) == 0 { - return - } - if _, exists := namespaces[podNamespace]; exists { - continue - } - namespaces[podNamespace] = true - // We require that all deployments have the desired number of replicas and are all up to date - if len(notReadyDeployments) != 0 { - ginkgo.Skip("Can not test when deployments are not ready") - } - gomega.Expect(notReadyDeployments).To(gomega.BeEmpty()) - ginkgo.By("Should return map of nodes to deployments") - nodesSorted := getDeploymentsNodes(podNamespace) - for _, n := range nodesSorted { - if _, exists := nodesNames[n.name]; !exists { - nodesNames[n.name] = n - } - } + + ginkgo.By(fmt.Sprintf("test deployment in namespace %s", env.NameSpaceUnderTest)) + deployments, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) + if len(deployments) == 0 { + return } + // We require that all deployments have the desired number of replicas and are all up to date + if len(notReadyDeployments) != 0 { + ginkgo.Skip("Can not test when deployments are not ready") + } + + ginkgo.By("Should return map of nodes to deployments") + nodesSorted := getDeploymentsNodes(env.NameSpaceUnderTest) + ginkgo.By("should create new replicas when node is drained") defer results.RecordResult(identifiers.TestPodRecreationIdentifier) - for _, n := range nodesNames { + for _, n := range nodesSorted { + closeOcSessions(env.ContainersUnderTest, n.name) + closeOcSessions(env.PartnerContainers, n.name) // drain node drainNode(n.name) // should go in this // verify deployments are ready again - for namespace := range namespaces { - _, notReadyDeployments = getDeployments(namespace) - if len(notReadyDeployments) != 0 { - uncordonNode(n.name) - ginkgo.Fail(fmt.Sprintf("did not create replicas when noede %s is drained", n.name)) - } + _, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) + if len(notReadyDeployments) != 0 { + uncordonNode(n.name) + ginkgo.Fail(fmt.Sprintf("did not create replicas when node %s is drained", n.name)) } + uncordonNode(n.name) } }) } +func closeOcSessions(containers map[configsections.ContainerIdentifier]*config.Container, nodeName string) { + for cid, c := range containers { + if cid.NodeName == nodeName { + log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) + c.Oc.Close() + delete(containers, cid) + } + } +} + type node struct { name string deployments map[string]bool From 213770d6eab1e12689ba7e27b9361b1d547b1151 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 27 Sep 2021 09:29:52 -0500 Subject: [PATCH 068/344] Get OCP version and save it in claim file (#365) Get OCP version and save it in claim file --- CATALOG.md | 14 +- go.mod | 3 +- go.sum | 17 +-- .../handlers/clusterversion/clusterversion.go | 131 ++++++++++++++++++ .../clusterversion/clusterversion_test.go | 121 ++++++++++++++++ pkg/tnf/handlers/clusterversion/doc.go | 18 +++ pkg/tnf/identifier/identifiers.go | 19 +++ test-network-function/diagnostic/suite.go | 29 +++- .../identifiers/identifiers.go | 11 ++ test-network-function/suite_test.go | 10 +- 10 files changed, 355 insertions(+), 18 deletions(-) create mode 100644 pkg/tnf/handlers/clusterversion/clusterversion.go create mode 100644 pkg/tnf/handlers/clusterversion/clusterversion_test.go create mode 100644 pkg/tnf/handlers/clusterversion/doc.go diff --git a/CATALOG.md b/CATALOG.md index 07c47b3df..7cd3b91c5 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -62,12 +62,12 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/operator-is-certified tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP). Result Type|normative Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP). -### http://test-network-function.com/testcases/diagnostic/cluster-csi-info +### http://test-network-function.com/testcases/diagnostic/clusterversion Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/diagnostic/cluster-csi-info extracts CSI driver information in the cluster. +Description|http://test-network-function.com/testcases/diagnostic/clusterversion extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| ### http://test-network-function.com/testcases/diagnostic/extract-node-information @@ -235,6 +235,16 @@ Suggested Remediation|Test failure indicates that the underlying Node's' kernel ## Test Case Building Blocks Catalog A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: +### http://test-network-function.com/tests/clusterVersion +Property|Description +---|--- +Version|v1.0.0 +Description|extracts OCP versions from the cluster +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`oc` + ### http://test-network-function.com/tests/clusterrolebinding Property|Description ---|--- diff --git a/go.mod b/go.mod index fca27800e..633410946 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,9 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 - github.com/test-network-function/test-network-function-claim v1.0.3 + github.com/test-network-function/test-network-function-claim v1.0.4 github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect google.golang.org/grpc v1.40.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 2192a0995..fb2f3adce 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -99,7 +100,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= @@ -133,17 +133,12 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 h1:/nu0LtOLsZMlqfk6i50C5fmo5cp5WfciWggZYlwGNAg= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= -github.com/google/goexpect v0.0.0-20210914193350-0010e15e87b9 h1:HOFqDJo8gSnYucPGdobe/LylQLOmW9XpZ3sML1YBtg8= -github.com/google/goexpect v0.0.0-20210914193350-0010e15e87b9/go.mod h1:3bkiyiIBwCvUXYv811zuwnHlrPotynhld5LviGDXPfw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= -github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4= -github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -187,7 +182,6 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -262,8 +256,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/test-network-function/test-network-function-claim v1.0.3 h1:r13sg1pOdbW095/IwF+eFW4jppbcyaOGBQ3CGK66j3o= -github.com/test-network-function/test-network-function-claim v1.0.3/go.mod h1:hDN1y2l8D7K3CX2ZyJThSCBXvVksDLJd1xOMmWqUlaY= +github.com/test-network-function/test-network-function-claim v1.0.4 h1:f/fT8fKHJifVvJ+koaLXbRzH8HG5TKrkwm67tt+dmY8= +github.com/test-network-function/test-network-function-claim v1.0.4/go.mod h1:hDN1y2l8D7K3CX2ZyJThSCBXvVksDLJd1xOMmWqUlaY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -322,6 +316,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -333,6 +328,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -446,8 +442,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg= -golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -513,6 +507,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/tnf/handlers/clusterversion/clusterversion.go b/pkg/tnf/handlers/clusterversion/clusterversion.go new file mode 100644 index 000000000..c5975ad3b --- /dev/null +++ b/pkg/tnf/handlers/clusterversion/clusterversion.go @@ -0,0 +1,131 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package clusterversion + +import ( + "regexp" + "time" + + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +const ( + verRegex = "(?s).+" + numVersionsOcp = 3 + numVersionsMinikube = 2 +) + +// TestMetadata holds OCP version strings and test metadata +type TestMetadata struct { + versions ClusterVersion + result int + timeout time.Duration + args []string +} + +// ClusterVersion holds OCP version strings +type ClusterVersion struct { + Ocp, Oc, K8s string +} + +// NewClusterVersion creates a new TestMetadata tnf.Test. +// Just gets the ocp version for client and server +func NewClusterVersion(timeout time.Duration) *TestMetadata { + args := []string{"oc", "version"} + return &TestMetadata{ + timeout: timeout, + result: tnf.ERROR, + args: args, + } +} + +// Args returns the command line args for the test. +func (ver *TestMetadata) Args() []string { + return ver.args +} + +// GetVersions returns OCP client version. +func (ver *TestMetadata) GetVersions() ClusterVersion { + return ver.versions +} + +// GetIdentifier returns the tnf.Test specific identifiesa. +func (ver *TestMetadata) GetIdentifier() identifier.Identifier { + return identifier.ClusterVersionIdentifier +} + +// Timeout returns the timeout in seconds for the test. +func (ver *TestMetadata) Timeout() time.Duration { + return ver.timeout +} + +// Result returns the test result. +func (ver *TestMetadata) Result() int { + return ver.result +} + +// ReelFirst returns a step which expects the ping statistics within the test timeout. +func (ver *TestMetadata) ReelFirst() *reel.Step { + return &reel.Step{ + Expect: []string{verRegex}, + Timeout: ver.timeout, + } +} + +func deleteEmpty(s []string) []string { + var r []string + for _, str := range s { + if str != "" { + r = append(r, str) + } + } + return r +} + +// ReelMatch ensures that list of nodes is not empty and stores the names as []string +func (ver *TestMetadata) ReelMatch(_, _, match string) *reel.Step { + re := regexp.MustCompile("(Server Version: )|(Client Version: )|(Kubernetes Version: )|(\n)") + versions := re.Split(match, -1) + versions = deleteEmpty(versions) + if len(versions) != numVersionsOcp && len(versions) != numVersionsMinikube { + ver.result = tnf.FAILURE + return nil + } + ver.result = tnf.SUCCESS + + if len(versions) == numVersionsOcp { + ver.versions.Oc = versions[0] + ver.versions.Ocp = versions[1] + ver.versions.K8s = versions[2] + } else { + ver.versions.Oc = versions[0] + ver.versions.Ocp = "n/a" + ver.versions.K8s = versions[1] + } + return nil +} + +// ReelTimeout does nothing; no action is necessary upon timeout. +func (ver *TestMetadata) ReelTimeout() *reel.Step { + return nil +} + +// ReelEOF does nothing; no action is necessary upon EOF. +func (ver *TestMetadata) ReelEOF() { +} diff --git a/pkg/tnf/handlers/clusterversion/clusterversion_test.go b/pkg/tnf/handlers/clusterversion/clusterversion_test.go new file mode 100644 index 000000000..1bacf5590 --- /dev/null +++ b/pkg/tnf/handlers/clusterversion/clusterversion_test.go @@ -0,0 +1,121 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package clusterversion_test + +import ( + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + ver "github.com/test-network-function/test-network-function/pkg/tnf/handlers/clusterversion" +) + +func Test_NewNodeNames(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + assert.Equal(t, testTimeoutDuration, newVer.Timeout()) + assert.Equal(t, newVer.Result(), tnf.ERROR) +} + +func Test_ReelFirstPositiveOcp(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + firstStep := newVer.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputSuccessOcp) + assert.Len(t, matches, 1) + assert.Equal(t, testInputSuccessOcp, matches[0]) +} + +func Test_ReelFirstPositiveMinikube(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + firstStep := newVer.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputSuccessMinikube) + assert.Len(t, matches, 1) + assert.Equal(t, testInputSuccessMinikube, matches[0]) +} + +func Test_ReelFirstPositiveEmpty(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + firstStep := newVer.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputFailure) + assert.Len(t, matches, 1) + assert.Equal(t, testInputFailure, matches[0]) +} + +func Test_ReelFirstNegative(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + firstStep := newVer.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputError) + assert.Len(t, matches, 0) +} + +func Test_ReelMatchSuccessOcp(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + step := newVer.ReelMatch("", "", testInputSuccessOcp) + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, newVer.Result()) + assert.Equal(t, newVer.GetVersions().Oc, "4.7.16") + assert.Equal(t, newVer.GetVersions().Ocp, "4.8.3") + assert.Equal(t, newVer.GetVersions().K8s, "v1.21.1+051ac4f") +} + +func Test_ReelMatchSuccessMinikube(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + step := newVer.ReelMatch("", "", testInputSuccessMinikube) + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, newVer.Result()) + assert.Equal(t, newVer.GetVersions().Oc, "4.7.16") + assert.Equal(t, newVer.GetVersions().Ocp, "n/a") + assert.Equal(t, newVer.GetVersions().K8s, "v1.21.1+051ac4f") +} + +func Test_ReelMatchFail(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + step := newVer.ReelMatch("", "", testInputFailure) + assert.Nil(t, step) + assert.Equal(t, tnf.FAILURE, newVer.Result()) + assert.Equal(t, newVer.GetVersions().Ocp, "") + assert.Equal(t, newVer.GetVersions().Oc, "") + assert.Equal(t, newVer.GetVersions().K8s, "") +} + +// Just ensure there are no panics. +func Test_ReelEof(t *testing.T) { + newVer := ver.NewClusterVersion(testTimeoutDuration) + assert.NotNil(t, newVer) + newVer.ReelEOF() +} + +const ( + testTimeoutDuration = time.Second * 2 + testInputError = "" + testInputFailure = "NAME\n" + testInputSuccessOcp = "Client Version: 4.7.16\nServer Version: 4.8.3\nKubernetes Version: v1.21.1+051ac4f\n" + testInputSuccessMinikube = "Client Version: 4.7.16\nKubernetes Version: v1.21.1+051ac4f\n" +) diff --git a/pkg/tnf/handlers/clusterversion/doc.go b/pkg/tnf/handlers/clusterversion/doc.go new file mode 100644 index 000000000..7dd2f7963 --- /dev/null +++ b/pkg/tnf/handlers/clusterversion/doc.go @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package clusterversion provides a test for reading the cluster's node names +package clusterversion diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 9909f6a13..4cbdfeba3 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -58,6 +58,7 @@ const ( shutdownIdentifierURL = "http://test-network-function.com/tests/shutdown" scalingIdentifierURL = "http://test-network-function.com/tests/scaling" csiDriverIdentifierURL = "http://test-network-function.com/tests/csiDriver" + clusterVersionIdentifierURL = "http://test-network-function.com/tests/clusterVersion" versionOne = "v1.0.0" ) @@ -596,6 +597,18 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.OcBinaryName, }, }, + clusterVersionIdentifierURL: { + Identifier: ClusterVersionIdentifier, + Description: "extracts OCP versions from the cluster", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + }, + }, } // HostnameIdentifier is the Identifier used to represent the generic hostname test case. @@ -832,3 +845,9 @@ var CSIDriverIdentifier = Identifier{ URL: csiDriverIdentifierURL, SemanticVersion: versionOne, } + +// ClusterVersionIdentifier is the Identifier used to represent the OCP versions test case. +var ClusterVersionIdentifier = Identifier{ + URL: clusterVersionIdentifierURL, + SemanticVersion: versionOne, +} diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 39f314af8..7bbe0276c 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -14,6 +14,7 @@ import ( ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/clusterversion" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodedebug" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" @@ -35,6 +36,8 @@ var ( cniPlugins = make([]CniPlugin, 0) + versionsOcp clusterversion.ClusterVersion + nodesHwInfo = NodesHwInfo{} // csiDriver stores the csi driver JSON output of `oc get csidriver -o json` @@ -65,7 +68,15 @@ var ( var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.DiagnosticTestKey) { ginkgo.When("a cluster is set up and installed with OpenShift", func() { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestExtractNodeInformationIdentifier) + + ginkgo.By("should report OCP version") + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestclusterVersionIdentifier) + ginkgo.It(testID, func() { + defer results.RecordResult(identifiers.TestclusterVersionIdentifier) + testOcpVersion() + }) + + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestExtractNodeInformationIdentifier) ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestExtractNodeInformationIdentifier) context := common.GetContext() @@ -146,6 +157,11 @@ func GetCniPlugins() []CniPlugin { return cniPlugins } +// GetVersionsOcp return OCP versions +func GetVersionsOcp() clusterversion.ClusterVersion { + return versionsOcp +} + // GetNodesHwInfo returns an object with HW info of one master and one worker func GetNodesHwInfo() NodesHwInfo { return nodesHwInfo @@ -199,6 +215,17 @@ func listNodeCniPlugins(nodeName string) []CniPlugin { return result } +func testOcpVersion() { + context := common.GetContext() + tester := clusterversion.NewClusterVersion(defaultTestTimeout) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + versionsOcp = tester.GetVersions() +} + func testCniPlugins() { if common.IsMinikube() { ginkgo.Skip("can't use 'oc debug' in minikube") diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index e3adf55fe..f3cccdc01 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -200,6 +200,11 @@ var ( Url: formTestURL(common.DiagnosticTestKey, "cluster-csi-info"), Version: versionOne, } + // TestclusterVersionIdentifier list Cluster CSIdriver Identifier retrieves Third Party CSI driver info. + TestclusterVersionIdentifier = claim.Identifier{ + Url: formTestURL(common.DiagnosticTestKey, "clusterversion"), + Version: versionOne, + } ) func formDescription(identifier claim.Identifier, description string) string { @@ -519,4 +524,10 @@ the changes for you.`, Description: formDescription(TestClusterCsiInfoIdentifier, `extracts CSI driver information in the cluster.`), }, + TestClusterCsiInfoIdentifier: { + Identifier: TestclusterVersionIdentifier, + Type: informativeResult, + Description: formDescription(TestclusterVersionIdentifier, + `Extracts OCP versions from the cluster.`), + }, } diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 750b28b7c..baeeca7ff 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -124,12 +124,12 @@ func TestTest(t *testing.T) { claimData := claimRoot.Claim claimData.Configurations = make(map[string]interface{}) claimData.Nodes = make(map[string]interface{}) - incorporateTNFVersion(claimData) // run the test suite ginkgo.RunSpecs(t, CnfCertificationTestSuiteName) endTime := time.Now() + incorporateVersions(claimData) // process the test results from this test suite, the cnf-features-deploy test suite, and any extra informational // messages. junitMap := make(map[string]interface{}) @@ -164,9 +164,13 @@ func getTNFVersion() *version.Version { } // incorporateTNFVersion adds the TNF version to the claim. -func incorporateTNFVersion(claimData *claim.Claim) { +func incorporateVersions(claimData *claim.Claim) { claimData.Versions = &claim.Versions{ - Tnf: getTNFVersion().Tag, + Tnf: getTNFVersion().Tag, + TnfGitCommit: GitCommit, + OcClient: diagnostic.GetVersionsOcp().Oc, + Ocp: diagnostic.GetVersionsOcp().Ocp, + K8s: diagnostic.GetVersionsOcp().K8s, } } From e8335f730b96c9c43b736437b2695d17f3978613 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Mon, 27 Sep 2021 14:51:20 -0400 Subject: [PATCH 069/344] simplify log output (#369) --- test-network-function/common/env.go | 3 +++ test-network-function/suite_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index 52ede320e..be27813af 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -100,14 +100,17 @@ func SetLogLevel() { // SetLogFormat sets the log format for logrus func SetLogFormat() { + log.Info("debug format initialization: start") customFormatter := new(log.TextFormatter) customFormatter.TimestampFormat = time.StampMilli customFormatter.PadLevelText = true customFormatter.FullTimestamp = true + customFormatter.ForceColors = true log.SetReportCaller(true) customFormatter.CallerPrettyfier = func(f *runtime.Frame) (string, string) { _, filename := path.Split(f.File) return strconv.Itoa(f.Line) + "]", fmt.Sprintf("[%s:", filename) } log.SetFormatter(customFormatter) + log.Error("debug format initialization: done") } diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index baeeca7ff..6ca511adc 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -114,9 +114,9 @@ func TestTest(t *testing.T) { // set up input flags and register failure handlers. flag.Parse() gomega.RegisterFailHandler(ginkgo.Fail) - log.Info("Version: ", programVersion, " ( ", GitCommit, " )") - common.SetLogLevel() common.SetLogFormat() + common.SetLogLevel() + log.Info("Version: ", programVersion, " ( ", GitCommit, " )") tnfcommon.OcDebugImageID = common.GetOcDebugImageID() // Initialize the claim with the start time, tnf version, etc. From 99fa148fc55f93b0a64aefa97e06ce26705aba50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 15:02:30 -0400 Subject: [PATCH 070/344] Bump google.golang.org/grpc from 1.40.0 to 1.41.0 (#368) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.40.0 to 1.41.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.40.0...v1.41.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jun Chen --- go.mod | 3 +-- go.sum | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 633410946..d285beb98 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.4 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - google.golang.org/grpc v1.40.0 + google.golang.org/grpc v1.41.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index fb2f3adce..2c18c8c26 100644 --- a/go.sum +++ b/go.sum @@ -41,7 +41,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/a-h/generate v0.0.0-20190312091541-e59c34d33fb3/go.mod h1:traiLYQ0YD7qUMCdjo6/jSaJRPHXniX4HVs+PhEhYpc= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -50,11 +49,13 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -62,7 +63,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -75,7 +76,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -85,10 +86,10 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -133,6 +134,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 h1:/nu0LtOLsZMlqfk6i50C5fmo5cp5WfciWggZYlwGNAg= github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= @@ -182,6 +184,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -191,8 +194,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -238,7 +243,6 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= @@ -316,7 +320,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -328,7 +331,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -507,7 +509,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -605,8 +606,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -621,6 +622,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From 04334151433312adcbe39c4eabc7df2d18f1284b Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 28 Sep 2021 15:28:05 -0400 Subject: [PATCH 071/344] Expose the partner image repo var (#370) * Update run-container.sh * Update README.md * Update README.md * Update README.md --- README.md | 9 ++++++--- script/run-container.sh | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5beea9e8b..886db51d7 100644 --- a/README.md +++ b/README.md @@ -129,13 +129,16 @@ Set this variable to deploy partner pods in a custom namespace instead of the de export TNF_PARTNER_NAMESPACE="CNF-ns" ``` -### Specify the image id to be used with oc debug commands +### Disconnected environment In disconnected environment, only specific versions of images are mirrored to the local repo. For the `oc debug` command (used by a number of tests) to work, set TNF_OC_DEBUG_IMAGE_ID: ```shell-script -export TNF_OC_DEBUG_IMAGE_ID="quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0f5ce898fbad3671fecd6f797130f950fb1abdbaf0d8154c22e2b59c74e3a918" +export TNF_OC_DEBUG_IMAGE_ID="registry.dfwt5g.lab:5000/openshift-release-dev/ocp-v4.0-art-dev@sha256:0f5ce898fbad3671fecd6f797130f950fb1abdbaf0d8154c22e2b59c74e3a918" +``` +For similar reasons, the partner pod image `quay.io/testnetworkfunction/cnf-test-partner` should be mirrored too and `TNF_PARTNER_REPO` should be set to the local repo, e.g.: +```shell-script +export TNF_PARTNER_REPO="registry.dfwt5g.lab:5000/testnetworkfunction" ``` - ### Execute test suites from openshift-kni/cnf-feature-deploy The test suites from openshift-kni/cnf-feature-deploy can be run prior to the actual CNF certification test execution and the results are incorporated in the same claim file if the following environment variable is set: diff --git a/script/run-container.sh b/script/run-container.sh index 38132f0af..fde51731d 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -108,6 +108,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ -e TNF_PARTNER_NAMESPACE=$TNF_PARTNER_NAMESPACE \ + -e TNF_PARTNER_REPO=$TNF_PARTNER_REPO \ -e TNF_OC_DEBUG_IMAGE_ID=$TNF_OC_DEBUG_IMAGE_ID \ -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ From 1749122cf911a4239361136fb30615adde160891 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 28 Sep 2021 16:15:21 -0400 Subject: [PATCH 072/344] Ignore terminating pods in auto discovery (#371) --- CATALOG.md | 2 +- pkg/config/autodiscover/autodiscover.go | 2 +- .../autodiscover/autodiscover_targets.go | 4 ++-- pkg/config/autodiscover/pod_info.go | 19 ++++++++++++++----- .../identifiers/identifiers.go | 4 ++-- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 7cd3b91c5..89bfed08c 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -211,7 +211,7 @@ Suggested Remediation|HugePage settings should be configured either directly thr Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release The test verifies if the container base image is redhat. +Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release verifies if the container base image is redhat. Result Type|normative Suggested Remediation|build a new docker image that's based on UBI (redhat universal base image). ### http://test-network-function.com/testcases/platform-alteration/sysctl-config diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index d49d7d871..74dab1fa7 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -75,7 +75,7 @@ func getContainersByLabel(label configsections.Label, namespace string) (contain return nil, err } for i := range pods.Items { - containers = append(containers, buildContainersFromPodResource(&pods.Items[i])...) + containers = append(containers, buildContainersFromPodResource(pods.Items[i])...) } return containers, nil } diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 419aaa31e..ec90ba5b9 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -41,8 +41,8 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa pods, err := GetPodsByLabel(l, namespace) if err == nil { for i := range pods.Items { - target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(&pods.Items[i])) - target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(&pods.Items[i])...) + target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(pods.Items[i])) + target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(pods.Items[i])...) } } else { log.Warnf("failed to query by label: %v %v", l, err) diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index b2bd61581..ae4f6cbc7 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -38,16 +38,17 @@ var ( // PodList holds the data from an `oc get pods -o json` command type PodList struct { - Items []PodResource `json:"items"` + Items []*PodResource `json:"items"` } // PodResource is a single Pod from an `oc get pods -o json` command type PodResource struct { Metadata struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - Labels map[string]string `json:"labels"` - Annotations map[string]string `json:"annotations"` + Name string `json:"name"` + Namespace string `json:"namespace"` + DeletionTimestamp string `json:"deletionTimestamp"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` } `json:"metadata"` Spec struct { ServiceAccount string `json:"serviceaccountname"` @@ -167,5 +168,13 @@ func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, err return nil, err } + // Filter out terminating pods + var pods []*PodResource + for _, pod := range podList.Items { + if pod.Metadata.DeletionTimestamp == "" { + pods = append(pods, pod) + } + } + podList.Items = pods return &podList, nil } diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index f3cccdc01..651079b2c 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -515,7 +515,7 @@ the changes for you.`, Identifier: TestIsRedHatReleaseIdentifier, Type: normativeResult, Description: formDescription(TestIsRedHatReleaseIdentifier, - `The test verifies if the container base image is redhat.`), + `verifies if the container base image is redhat.`), Remediation: `build a new docker image that's based on UBI (redhat universal base image).`, }, TestClusterCsiInfoIdentifier: { @@ -528,6 +528,6 @@ the changes for you.`, Identifier: TestclusterVersionIdentifier, Type: informativeResult, Description: formDescription(TestclusterVersionIdentifier, - `Extracts OCP versions from the cluster.`), + `extracts OCP versions from the cluster.`), }, } From 10d17957809c60a3b1cf1adc5360bae5a87e52b3 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 29 Sep 2021 14:51:46 -0400 Subject: [PATCH 073/344] Stop error channel monitoring routing prior to scaling in (#374) * Stop error channel monitoring routing prior to scaling in * Put the utility functions related to closing sessions together --- pkg/config/config.go | 5 +--- test-network-function/lifecycle/suite.go | 37 ++++++++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 5d1ac440a..06305b2fb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -186,8 +186,6 @@ func (env *TestEnvironment) doAutodiscover() { } autodiscover.FindTestPartner(&env.Config.Partner, env.NameSpaceUnderTest) - log.Infof("Test Configuration: %+v", *env) - env.ContainersToExcludeFromConnectivityTests = make(map[configsections.ContainerIdentifier]interface{}) for _, cid := range env.Config.ExcludeContainersFromConnectivityTests { @@ -199,8 +197,7 @@ func (env *TestEnvironment) doAutodiscover() { env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest env.OperatorsUnderTest = env.Config.Operators - log.Info(env.TestOrchestrator) - log.Info(env.ContainersUnderTest) + log.Infof("Test Configuration: %+v", *env) env.needsRefresh = false } diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 1da3e3ed3..90109873f 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -20,6 +20,7 @@ import ( "fmt" "path" "sort" + "strings" "time" "github.com/test-network-function/test-network-function/pkg/config" @@ -141,6 +142,26 @@ func restoreDeployments(env *config.TestEnvironment) { } } +func closeOcSessionsByDeployment(containers map[configsections.ContainerIdentifier]*config.Container, deployment configsections.Deployment) { + for cid, c := range containers { + if cid.Namespace == deployment.Namespace && strings.HasPrefix(cid.PodName, deployment.Name+"-") { + log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) + c.Oc.Close() + delete(containers, cid) + } + } +} + +func closeOcSessionsByNode(containers map[configsections.ContainerIdentifier]*config.Container, nodeName string) { + for cid, c := range containers { + if cid.NodeName == nodeName { + log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) + c.Oc.Close() + delete(containers, cid) + } + } +} + // runScalingTest Runs a Scaling handler TC and waits for all the deployments to be ready. func runScalingTest(deployment configsections.Deployment) { handler := scaling.NewScaling(common.DefaultTimeout, deployment.Namespace, deployment.Name, deployment.Replicas) @@ -167,6 +188,8 @@ func testScaling(env *config.TestEnvironment) { ginkgo.By(fmt.Sprintf("Scaling Deployment=%s, Replicas=%d (ns=%s)", deployment.Name, deployment.Replicas, deployment.Namespace)) + closeOcSessionsByDeployment(env.ContainersUnderTest, deployment) + replicaCount := deployment.Replicas // ScaleIn, removing one pod from the replicaCount @@ -297,8 +320,8 @@ func testPodsRecreation(env *config.TestEnvironment) { ginkgo.By("should create new replicas when node is drained") defer results.RecordResult(identifiers.TestPodRecreationIdentifier) for _, n := range nodesSorted { - closeOcSessions(env.ContainersUnderTest, n.name) - closeOcSessions(env.PartnerContainers, n.name) + closeOcSessionsByNode(env.ContainersUnderTest, n.name) + closeOcSessionsByNode(env.PartnerContainers, n.name) // drain node drainNode(n.name) // should go in this // verify deployments are ready again @@ -313,16 +336,6 @@ func testPodsRecreation(env *config.TestEnvironment) { }) } -func closeOcSessions(containers map[configsections.ContainerIdentifier]*config.Container, nodeName string) { - for cid, c := range containers { - if cid.NodeName == nodeName { - log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) - c.Oc.Close() - delete(containers, cid) - } - } -} - type node struct { name string deployments map[string]bool From c489eb678e3c00c20dba6c0cfdfc11c98d10549a Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 29 Sep 2021 14:54:04 -0500 Subject: [PATCH 074/344] Fix typo (#375) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 886db51d7..5dc188c13 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ export TNF_PARTNER_SRC_DIR=/home/userid/code/cnf-certification-test-partner When this variable is set, the run-cnf-suites.sh script will deploy/refresh the partner deployments/pods in the cluster before starting the test run. ### Specify the target namespace for partner pod deployment -Set this variable to deploy partner pods in a custom namespace instead of the default `tnf` namesapce. +Set this variable to deploy partner pods in a custom namespace instead of the default `tnf` namespace. ```shell-script export TNF_PARTNER_NAMESPACE="CNF-ns" From d40959a6f1cb182866deb70c1e2ab9248aba1aba Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Fri, 1 Oct 2021 09:18:54 -0500 Subject: [PATCH 075/344] Go 1.16 Upgrade (#366) --- Dockerfile | 2 +- README.md | 2 +- cmd/catalog/cmd/generate.go | 3 +-- cmd/claim/main.go | 7 +++---- go.mod | 2 +- pkg/config/autodiscover/csv_info_test.go | 4 ++-- pkg/config/autodiscover/deployment_info_test.go | 4 ++-- pkg/config/autodiscover/pod_info_test.go | 4 ++-- pkg/config/config.go | 3 +-- pkg/config/configsections/certified_request_test.go | 4 ++-- pkg/config/configsections/config_section_test.go | 4 ++-- pkg/jsonschema/utils.go | 6 +++--- pkg/tnf/handlers/generic/assertion/assertions_test.go | 4 ++-- pkg/tnf/handlers/generic/generic.go | 8 ++++---- pkg/tnf/handlers/generic/resultcontext_test.go | 4 ++-- pkg/tnf/handlers/ipaddr/ipaddr_test.go | 4 ++-- pkg/tnf/handlers/ping/ping_test.go | 4 ++-- pkg/tnf/identifier/identifier_test.go | 4 ++-- pkg/tnf/interactive/pty.go | 8 ++++---- pkg/tnf/testcases/base.go | 3 +-- pkg/tnf/testcases/base_test.go | 2 +- test-network-function/suite_test.go | 3 +-- test-network-function/version/version.go | 4 ++-- 23 files changed, 44 insertions(+), 49 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58abc9e33..436206437 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN yum install -y gcc git jq make wget # Install Go binary ENV GO_DL_URL="https://golang.org/dl" -ENV GO_BIN_TAR="go1.14.12.linux-amd64.tar.gz" +ENV GO_BIN_TAR="go1.16.8.linux-amd64.tar.gz" ENV GO_BIN_URL_x86_64=${GO_DL_URL}/${GO_BIN_TAR} ENV GOPATH="/root/go" RUN if [[ "$(uname -m)" -eq "x86_64" ]] ; then \ diff --git a/README.md b/README.md index 5dc188c13..b955691f0 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ At a minimum, the following dependencies must be installed *prior* to running `m Dependency|Minimum Version ---|--- -[GoLang](https://golang.org/dl/)|1.14 +[GoLang](https://golang.org/dl/)|1.16 [golangci-lint](https://golangci-lint.run/usage/install/)|1.32.2 [jq](https://stedolan.github.io/jq/)|1.6 [OpenShift Client](https://docs.openshift.com/container-platform/4.4/welcome/index.html)|4.4 diff --git a/cmd/catalog/cmd/generate.go b/cmd/catalog/cmd/generate.go index e3eca3f9a..80a14075c 100644 --- a/cmd/catalog/cmd/generate.go +++ b/cmd/catalog/cmd/generate.go @@ -19,7 +19,6 @@ package cmd import ( "encoding/json" "fmt" - "io/ioutil" "os" "path" "sort" @@ -115,7 +114,7 @@ func cmdJoin(elems []string, sep string) string { // emitTextFromFile is a utility method to stream file contents to stdout. This allows more natural specification of // the non-dynamic aspects of CATALOG.md. func emitTextFromFile(filename string) error { - text, err := ioutil.ReadFile(filename) + text, err := os.ReadFile(filename) if err != nil { return err } diff --git a/cmd/claim/main.go b/cmd/claim/main.go index 1e95145cb..a31f536fd 100644 --- a/cmd/claim/main.go +++ b/cmd/claim/main.go @@ -24,7 +24,6 @@ import ( "github.com/test-network-function/test-network-function-claim/pkg/claim" "github.com/test-network-function/test-network-function/pkg/junit" - "io/ioutil" "log" "os" "path/filepath" @@ -89,7 +88,7 @@ func main() { func claimUpdate() { fileUpdated := false - dat, err := ioutil.ReadFile(*claimFileTextPtr) + dat, err := os.ReadFile(*claimFileTextPtr) if err != nil { log.Fatalf("Error reading claim file :%v", err) } @@ -97,7 +96,7 @@ func claimUpdate() { claimRoot := readClaim(&dat) junitMap := claimRoot.Claim.RawResults - items, _ := ioutil.ReadDir(*reportFilesTextPtr) + items, _ := os.ReadDir(*reportFilesTextPtr) for _, item := range items { fileName := item.Name() @@ -119,7 +118,7 @@ func claimUpdate() { if err != nil { log.Fatalf("Failed to generate the claim: %v", err) } - err = ioutil.WriteFile(*claimFileTextPtr, payload, claimFilePermissions) + err = os.WriteFile(*claimFileTextPtr, payload, claimFilePermissions) if err != nil { log.Fatalf("Error writing claim data:\n%s", string(payload)) } diff --git a/go.mod b/go.mod index d285beb98..e27a77bb7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/test-network-function/test-network-function -go 1.14 +go 1.16 require ( github.com/Masterminds/semver/v3 v3.1.1 diff --git a/pkg/config/autodiscover/csv_info_test.go b/pkg/config/autodiscover/csv_info_test.go index 3011a806e..f7d05fd7e 100644 --- a/pkg/config/autodiscover/csv_info_test.go +++ b/pkg/config/autodiscover/csv_info_test.go @@ -18,8 +18,8 @@ package autodiscover import ( "encoding/json" - "io/ioutil" "log" + "os" "path" "testing" @@ -35,7 +35,7 @@ var ( ) func loadCSVResource(filePath string) (csv CSVResource) { - contents, err := ioutil.ReadFile(filePath) + contents, err := os.ReadFile(filePath) if err != nil { log.Fatalf("error (%s) loading CSVResource %s for testing", err, filePath) } diff --git a/pkg/config/autodiscover/deployment_info_test.go b/pkg/config/autodiscover/deployment_info_test.go index b15a0b8b0..192fba808 100644 --- a/pkg/config/autodiscover/deployment_info_test.go +++ b/pkg/config/autodiscover/deployment_info_test.go @@ -18,8 +18,8 @@ package autodiscover import ( "encoding/json" - "io/ioutil" "log" + "os" "path" "testing" @@ -35,7 +35,7 @@ var ( ) func loadDeployment(filePath string) (deployment DeploymentResource) { - contents, err := ioutil.ReadFile(filePath) + contents, err := os.ReadFile(filePath) if err != nil { log.Fatalf("error (%s) loading DeploymentResource %s for testing", err, filePath) } diff --git a/pkg/config/autodiscover/pod_info_test.go b/pkg/config/autodiscover/pod_info_test.go index e8c46d6b0..6e34eb753 100644 --- a/pkg/config/autodiscover/pod_info_test.go +++ b/pkg/config/autodiscover/pod_info_test.go @@ -18,8 +18,8 @@ package autodiscover import ( "encoding/json" - "io/ioutil" "log" + "os" "path" "testing" @@ -38,7 +38,7 @@ var ( ) func loadPodResource(filePath string) (pod PodResource) { - contents, err := ioutil.ReadFile(filePath) + contents, err := os.ReadFile(filePath) if err != nil { log.Fatalf("error (%s) loading PodResource %s for testing", err, filePath) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 06305b2fb..0102179ff 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,7 +18,6 @@ package config import ( "fmt" - "io/ioutil" "os" "time" @@ -145,7 +144,7 @@ func (env *TestEnvironment) loadConfigFromFile(filePath string) error { } log.Info("Loading config from file: ", filePath) - contents, err := ioutil.ReadFile(filePath) + contents, err := os.ReadFile(filePath) if err != nil { return err } diff --git a/pkg/config/configsections/certified_request_test.go b/pkg/config/configsections/certified_request_test.go index 6559c5a3a..0f44416d8 100644 --- a/pkg/config/configsections/certified_request_test.go +++ b/pkg/config/configsections/certified_request_test.go @@ -78,7 +78,7 @@ func setupRequestTest(marshalFun marshalFunc) (tempfileName string) { // loadRequestConfig reads `tmpPath`, unmarshals it using `unmarshalFun`, and returns the resulting `TestConfiguration`. func loadRequestConfig(tmpPath string, unmarshalFun unmarshalFunc) (conf *TestConfiguration) { - contents, err := ioutil.ReadFile(tmpPath) + contents, err := os.ReadFile(tmpPath) if err != nil { log.Fatal(err) } @@ -98,7 +98,7 @@ func saveRequestConfig(marshalFun marshalFunc, c *TestConfiguration, configPath if err != nil { log.Fatal(err) } - err = ioutil.WriteFile(configPath, bytes, filePerm) + err = os.WriteFile(configPath, bytes, filePerm) if err != nil { log.Fatal(err) } diff --git a/pkg/config/configsections/config_section_test.go b/pkg/config/configsections/config_section_test.go index 2d34e4f19..9f15319da 100644 --- a/pkg/config/configsections/config_section_test.go +++ b/pkg/config/configsections/config_section_test.go @@ -74,7 +74,7 @@ func saveConfig(c *TestConfiguration, configPath string) (err error) { if err != nil { return } - err = ioutil.WriteFile(configPath, bytes, filePerm) + err = os.WriteFile(configPath, bytes, filePerm) return } @@ -83,7 +83,7 @@ func saveConfigAsJSON(c *TestConfiguration, configPath string) (err error) { if err != nil { return } - err = ioutil.WriteFile(configPath, bytes, filePerm) + err = os.WriteFile(configPath, bytes, filePerm) return } diff --git a/pkg/jsonschema/utils.go b/pkg/jsonschema/utils.go index 985ba4ab9..3aa2f62e3 100644 --- a/pkg/jsonschema/utils.go +++ b/pkg/jsonschema/utils.go @@ -17,14 +17,14 @@ package jsonschema import ( - "io/ioutil" + "os" "github.com/xeipuuv/gojsonschema" ) // ValidateJSONFileAgainstSchema validates a given file against the supplied JSON schema. func ValidateJSONFileAgainstSchema(filename, schemaPath string) (*gojsonschema.Result, error) { - inputBytes, err := ioutil.ReadFile(filename) + inputBytes, err := os.ReadFile(filename) if err != nil { return nil, err } @@ -33,7 +33,7 @@ func ValidateJSONFileAgainstSchema(filename, schemaPath string) (*gojsonschema.R // ValidateJSONAgainstSchema validates a given byte array against the supplied JSON schema. func ValidateJSONAgainstSchema(inputBytes []byte, schemaPath string) (*gojsonschema.Result, error) { - schemaBytes, err := ioutil.ReadFile(schemaPath) + schemaBytes, err := os.ReadFile(schemaPath) if err != nil { return nil, err } diff --git a/pkg/tnf/handlers/generic/assertion/assertions_test.go b/pkg/tnf/handlers/generic/assertion/assertions_test.go index f1244ef5a..50cdcd8af 100644 --- a/pkg/tnf/handlers/generic/assertion/assertions_test.go +++ b/pkg/tnf/handlers/generic/assertion/assertions_test.go @@ -18,7 +18,7 @@ package assertion_test import ( "encoding/json" - "io/ioutil" + "os" "path" "regexp" "testing" @@ -131,7 +131,7 @@ func getTestFile(testName string) string { // TestAssertions_UnmarshalJSON also tests Assertion.UnmarshalJSON. func TestAssertions_UnmarshalJSON(t *testing.T) { for testName, testCase := range assertionsTestCases { - contents, err := ioutil.ReadFile(path.Join(getTestFile(testName))) + contents, err := os.ReadFile(path.Join(getTestFile(testName))) assert.Nil(t, err) assert.NotNil(t, contents) diff --git a/pkg/tnf/handlers/generic/generic.go b/pkg/tnf/handlers/generic/generic.go index 75d6ffdaa..47592aa54 100644 --- a/pkg/tnf/handlers/generic/generic.go +++ b/pkg/tnf/handlers/generic/generic.go @@ -19,7 +19,7 @@ package generic import ( "bytes" "encoding/json" - "io/ioutil" + "os" "regexp" "text/template" "time" @@ -195,7 +195,7 @@ func (g *Generic) ReelEOF() { // NewGenericFromJSONFile instantiates and initializes a Generic from a JSON-serialized file. func NewGenericFromJSONFile(filename, schemaPath string) (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { - inputBytes, err := ioutil.ReadFile(filename) + inputBytes, err := os.ReadFile(filename) if err != nil { return nil, nil, nil, err } @@ -220,7 +220,7 @@ func newGenericFromJSON(inputBytes []byte, schemaPath string) (*tnf.Tester, []re // suites. If the supplied template/values do not conform to the generic-test.schema.json schema, creation fails and // the result is returned to the caller for further inspection. func NewGenericFromTemplate(templateFile, schemaPath, valuesFile string) (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { - tplBytes, err := ioutil.ReadFile(valuesFile) + tplBytes, err := os.ReadFile(valuesFile) if err != nil { return nil, nil, nil, err } @@ -238,7 +238,7 @@ func NewGenericFromTemplate(templateFile, schemaPath, valuesFile string) (*tnf.T // suites. If the supplied values do not conform to the generic-test.schema.json schema, creation fails and the result // is returned to the caller for further inspection. func NewGenericFromMap(templateFile, schemaPath string, values map[string]interface{}) (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { - templateBytes, err := ioutil.ReadFile(templateFile) + templateBytes, err := os.ReadFile(templateFile) if err != nil { return nil, nil, nil, err } diff --git a/pkg/tnf/handlers/generic/resultcontext_test.go b/pkg/tnf/handlers/generic/resultcontext_test.go index 67029747c..bcbb67e72 100644 --- a/pkg/tnf/handlers/generic/resultcontext_test.go +++ b/pkg/tnf/handlers/generic/resultcontext_test.go @@ -18,7 +18,7 @@ package generic_test import ( "encoding/json" - "io/ioutil" + "os" "path" "testing" @@ -90,7 +90,7 @@ func TestResultContext_MarshalJSON(t *testing.T) { actualContents, err := json.MarshalIndent(resultContext, "", " ") assert.Nil(t, err) // Compare against an expected rendering which has been pre-verified. - expectedContents, err := ioutil.ReadFile(testFileName) + expectedContents, err := os.ReadFile(testFileName) assert.Nil(t, err) assert.Equal(t, string(expectedContents), string(actualContents)) } diff --git a/pkg/tnf/handlers/ipaddr/ipaddr_test.go b/pkg/tnf/handlers/ipaddr/ipaddr_test.go index c7478b288..dcfd4cafc 100644 --- a/pkg/tnf/handlers/ipaddr/ipaddr_test.go +++ b/pkg/tnf/handlers/ipaddr/ipaddr_test.go @@ -18,7 +18,7 @@ package ipaddr_test import ( "fmt" - "io/ioutil" + "os" "path" "testing" "time" @@ -63,7 +63,7 @@ func getMockOutputFilename(testName string) string { func getMockOutput(t *testing.T, testName string) string { fileName := getMockOutputFilename(testName) - b, err := ioutil.ReadFile(fileName) + b, err := os.ReadFile(fileName) assert.Nil(t, err) return string(b) } diff --git a/pkg/tnf/handlers/ping/ping_test.go b/pkg/tnf/handlers/ping/ping_test.go index abd7d0f54..3c0983d1e 100644 --- a/pkg/tnf/handlers/ping/ping_test.go +++ b/pkg/tnf/handlers/ping/ping_test.go @@ -18,7 +18,7 @@ package ping_test import ( "fmt" - "io/ioutil" + "os" "path" "strconv" "testing" @@ -102,7 +102,7 @@ func getMockOutputFilename(testName string) string { func getMockOutput(t *testing.T, testName string) string { fileName := getMockOutputFilename(testName) - b, err := ioutil.ReadFile(fileName) + b, err := os.ReadFile(fileName) assert.Nil(t, err) return string(b) } diff --git a/pkg/tnf/identifier/identifier_test.go b/pkg/tnf/identifier/identifier_test.go index 5e8f9e5a4..5011ea0ab 100644 --- a/pkg/tnf/identifier/identifier_test.go +++ b/pkg/tnf/identifier/identifier_test.go @@ -18,7 +18,7 @@ package identifier_test import ( "encoding/json" - "io/ioutil" + "os" "path" "testing" @@ -69,7 +69,7 @@ func getTestFile(testName string) string { func TestIdentifier_UnmarshalJSON(t *testing.T) { for testName, testCase := range testIdentifierUnmarshalJSONTestCases { testFile := getTestFile(testName) - contents, err := ioutil.ReadFile(testFile) + contents, err := os.ReadFile(testFile) assert.Nil(t, err) assert.NotNil(t, contents) diff --git a/pkg/tnf/interactive/pty.go b/pkg/tnf/interactive/pty.go index 32c48b27b..35584c050 100644 --- a/pkg/tnf/interactive/pty.go +++ b/pkg/tnf/interactive/pty.go @@ -19,7 +19,7 @@ package interactive import ( "bytes" "html/template" - "io/ioutil" + "os" "time" "github.com/test-network-function/test-network-function/pkg/jsonschema" @@ -37,7 +37,7 @@ const ( // entry-point, which will vary for unit tests, executables, and test suites. If the supplied file does not conform to // the generic-pty.schema.json schema, creation fails and the result is returned to the caller for further inspection. func SpawnGenericPTYFromYAMLFile(ptyPath, schemaPath string, spawner *Spawner) (*Context, *gojsonschema.Result, error) { - ptyBytes, err := ioutil.ReadFile(ptyPath) + ptyBytes, err := os.ReadFile(ptyPath) if err != nil { return nil, nil, err } @@ -66,7 +66,7 @@ func SpawnGenericPTYFromYAML(inputBytes []byte, schemaPath string, spawner *Spaw // suites. If the supplied template/values do not conform to the generic-pty.schema.json schema, creation fails and the // result is returned to the caller for further inspection. func SpawnGenericPTYFromYAMLTemplate(templateFile, valuesFile, schemaPath string, spawner *Spawner) (*Context, *gojsonschema.Result, error) { - tplBytes, err := ioutil.ReadFile(valuesFile) + tplBytes, err := os.ReadFile(valuesFile) if err != nil { return nil, nil, err } @@ -78,7 +78,7 @@ func SpawnGenericPTYFromYAMLTemplate(templateFile, valuesFile, schemaPath string return nil, nil, err } - templateBytes, err := ioutil.ReadFile(templateFile) + templateBytes, err := os.ReadFile(templateFile) if err != nil { return nil, nil, err } diff --git a/pkg/tnf/testcases/base.go b/pkg/tnf/testcases/base.go index 572fab679..77f5df053 100644 --- a/pkg/tnf/testcases/base.go +++ b/pkg/tnf/testcases/base.go @@ -18,7 +18,6 @@ package testcases import ( "encoding/json" - "io/ioutil" "os" "regexp" "strings" @@ -233,7 +232,7 @@ type ConfiguredTest struct { // LoadConfiguredTestFile loads configured test cases to struct func LoadConfiguredTestFile(filepath string) (c *ConfiguredTestCase, err error) { - yamlFile, err := ioutil.ReadFile(filepath) + yamlFile, err := os.ReadFile(filepath) if err != nil { return } diff --git a/pkg/tnf/testcases/base_test.go b/pkg/tnf/testcases/base_test.go index 6cb8f904d..b061f5059 100644 --- a/pkg/tnf/testcases/base_test.go +++ b/pkg/tnf/testcases/base_test.go @@ -58,7 +58,7 @@ func setup() { log.Fatal(err) } bytes, _ := yaml.Marshal(testConfigure) - err = ioutil.WriteFile(file.Name(), bytes, filePerm) + err = os.WriteFile(file.Name(), bytes, filePerm) if err != nil { log.Fatal(err) } diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 6ca511adc..4c525c744 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -19,7 +19,6 @@ package suite import ( j "encoding/json" "flag" - "io/ioutil" "os" "path/filepath" "testing" @@ -224,7 +223,7 @@ func marshalClaimOutput(claimRoot *claim.Root) []byte { // writeClaimOutput writes the output payload to the claim file. In the event of an error, this method fatally fails. func writeClaimOutput(claimOutputFile string, payload []byte) { - err := ioutil.WriteFile(claimOutputFile, payload, claimFilePermissions) + err := os.WriteFile(claimOutputFile, payload, claimFilePermissions) if err != nil { log.Fatalf("Error writing claim data:\n%s", string(payload)) } diff --git a/test-network-function/version/version.go b/test-network-function/version/version.go index c0112d620..d7b064324 100644 --- a/test-network-function/version/version.go +++ b/test-network-function/version/version.go @@ -18,7 +18,7 @@ package version import ( "encoding/json" - "io/ioutil" + "os" "path" ) @@ -34,7 +34,7 @@ type Version struct { // GetVersion extracts the test-network-function version. func GetVersion() (*Version, error) { - contents, err := ioutil.ReadFile(defaultVersionFile) + contents, err := os.ReadFile(defaultVersionFile) if err != nil { return nil, err } From 7227b99802da831af336daf7f0d8c80f0e3babfd Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Tue, 5 Oct 2021 16:31:01 +0300 Subject: [PATCH 076/344] =?UTF-8?q?CNFCERT-92:Extend=20auto=20discovery=20?= =?UTF-8?q?process=20to=20build=20a=20node=20list=20and=20upd=E2=80=A6=20(?= =?UTF-8?q?#373)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend auto discovery process to build a node list and update node level tests --- go.mod | 1 + go.sum | 4 ++ .../autodiscover/autodiscover_targets.go | 49 +++++++++++++++ pkg/config/config.go | 4 ++ pkg/config/configsections/Node.go | 49 +++++++++++++++ pkg/config/configsections/common.go | 2 + test-network-function/diagnostic/suite.go | 44 ++++++------- test-network-function/platform/suite.go | 61 ++++++------------- 8 files changed, 147 insertions(+), 67 deletions(-) create mode 100644 pkg/config/configsections/Node.go diff --git a/go.mod b/go.mod index e27a77bb7..97b48a805 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.4 github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect google.golang.org/grpc v1.41.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 2c18c8c26..06ae2a522 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -320,6 +321,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -331,6 +333,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -509,6 +512,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index ec90ba5b9..569d5ddde 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -19,7 +19,11 @@ package autodiscover import ( log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/test-network-function/common" ) const ( @@ -66,6 +70,51 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa } target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target, namespace)...) + target.Nodes = GetNodesList() +} + +// GetNodesList Function that return a list of node and what is the type of them. +func GetNodesList() (nodes map[string]configsections.Node) { + nodes = make(map[string]configsections.Node) + var nodeNames []string + context := common.GetContext() + tester := nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{configsections.MasterLabel: nil}) + test, _ := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + _, err := test.Run() + if err != nil { + log.Error("Unable to get node list ", ". Error: ", err) + return + } + nodeNames = tester.GetNodeNames() + for i := range nodeNames { + nodes[nodeNames[i]] = configsections.Node{ + Name: nodeNames[i], + Labels: []string{configsections.MasterLabel}, + } + } + + context = common.GetContext() + tester = nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{configsections.WorkerLabel: nil}) + test, _ = tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + _, err = test.Run() + if err != nil { + log.Error("Unable to get node list ", ". Error: ", err) + } else { + nodeNames = tester.GetNodeNames() + for i := range nodeNames { + if _, ok := nodes[nodeNames[i]]; ok { + var node = nodes[nodeNames[i]] + node.Labels = append(node.Labels, configsections.WorkerLabel) + nodes[nodeNames[i]] = node + } else { + nodes[nodeNames[i]] = configsections.Node{ + Name: nodeNames[i], + Labels: []string{configsections.WorkerLabel}, + } + } + } + } + return nodes } // FindTestDeployments uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, diff --git a/pkg/config/config.go b/pkg/config/config.go index 0102179ff..a655fbb81 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -126,6 +126,8 @@ type TestEnvironment struct { DeploymentsUnderTest []configsections.Deployment OperatorsUnderTest []configsections.Operator NameSpaceUnderTest string + Nodes map[string]configsections.Node + // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -196,7 +198,9 @@ func (env *TestEnvironment) doAutodiscover() { env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest env.OperatorsUnderTest = env.Config.Operators + env.Nodes = env.Config.Nodes log.Infof("Test Configuration: %+v", *env) + env.needsRefresh = false } diff --git a/pkg/config/configsections/Node.go b/pkg/config/configsections/Node.go new file mode 100644 index 000000000..562167da4 --- /dev/null +++ b/pkg/config/configsections/Node.go @@ -0,0 +1,49 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +// WorkerLabel const for k8s worker +const WorkerLabel = "node-role.kubernetes.io/worker" + +// MasterLabel const for k8s for master +const MasterLabel = "node-role.kubernetes.io/master" + +// Node defines in the cluster. with name of the node and the type of this node master/worker,,,,. +type Node struct { + Name string + Labels []string +} + +// IsMaster Function that return if the node is master +func (node Node) IsMaster() bool { + for _, t := range node.Labels { + if t == MasterLabel { + return true + } + } + return false +} + +// IsWorker Function that return if the node is worker +func (node Node) IsWorker() bool { + for _, t := range node.Labels { + if t == WorkerLabel { + return true + } + } + return false +} diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 2bd7d5816..f355d9fb7 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -82,4 +82,6 @@ type TestTarget struct { ExcludeContainersFromConnectivityTests []ContainerIdentifier `yaml:"excludeContainersFromConnectivityTests" json:"excludeContainersFromConnectivityTests"` // Operator is the list of operator objects that needs to be tested. Operators []Operator `yaml:"operators,omitempty" json:"operators,omitempty"` + // Node list + Nodes map[string]Node `yaml:"Nodes" json:"Nodes"` } diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 7bbe0276c..6d40263a4 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" "github.com/test-network-function/test-network-function/test-network-function/results" @@ -17,7 +18,6 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/clusterversion" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodedebug" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" ) @@ -172,27 +172,22 @@ func GetCsiDriverInfo() map[string]interface{} { return csiDriver } -func getFirstNode(labelFilter map[string]*string) string { - context := common.GetContext() - tester := nodenames.NewNodeNames(defaultTestTimeout, labelFilter) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - nodeNames := tester.GetNodeNames() - gomega.Expect(nodeNames).NotTo(gomega.BeEmpty()) - return nodeNames[0] -} - -func getMasterNodeName() string { - const masterNodeLabel = "node-role.kubernetes.io/master" - return getFirstNode(map[string]*string{masterNodeLabel: nil}) +func getMasterNodeName(env *config.TestEnvironment) string { + for _, node := range env.Nodes { + if node.IsMaster() { + return node.Name + } + } + return "" } -func getWorkerNodeName() string { - const workerNodeLabel = "node-role.kubernetes.io/worker" - return getFirstNode(map[string]*string{workerNodeLabel: nil}) +func getWorkerNodeName(env *config.TestEnvironment) string { + for _, node := range env.Nodes { + if node.IsWorker() { + return node.Name + } + } + return "" } func listNodeCniPlugins(nodeName string) []CniPlugin { @@ -231,7 +226,8 @@ func testCniPlugins() { ginkgo.Skip("can't use 'oc debug' in minikube") } // get name of a master node - nodeName := getMasterNodeName() + env := config.GetTestEnvironment() + nodeName := getMasterNodeName(env) gomega.Expect(nodeName).ToNot(gomega.BeEmpty()) // get CNI plugins from node cniPlugins = listNodeCniPlugins(nodeName) @@ -242,10 +238,10 @@ func testNodesHwInfo() { if common.IsMinikube() { ginkgo.Skip("can't use 'oc debug' in minikube") } - - masterNodeName := getMasterNodeName() + env := config.GetTestEnvironment() + masterNodeName := getMasterNodeName(env) gomega.Expect(masterNodeName).ToNot(gomega.BeEmpty()) - workerNodeName := getWorkerNodeName() + workerNodeName := getWorkerNodeName(env) gomega.Expect(workerNodeName).ToNot(gomega.BeEmpty()) nodesHwInfo.Master.NodeName = masterNodeName nodesHwInfo.Master.Lscpu = getNodeLscpu(masterNodeName) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index f8b3390d7..28708a1fc 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -41,7 +41,6 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/mckernelarguments" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodehugepages" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodemcname" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodetainted" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/podnodename" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/readbootconfig" @@ -65,8 +64,8 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { // use this boolean to turn off tests that require OS packages if !common.IsMinikube() { testContainersFsDiff(env) - testTainted() - testHugepages() + testTainted(env) + testHugepages(env) testBootParams(env) testSysctlConfigs(env) } @@ -287,66 +286,39 @@ func testSysctlConfigsHelper(podName, podNamespace string) { } } -func testTainted() { - var nodeNames []string +func testTainted(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) ginkgo.It(testID, func() { ginkgo.By("Testing tainted nodes in cluster") - ginkgo.By("Should return list of node names") - context := common.GetContext() - tester := nodenames.NewNodeNames(common.DefaultTimeout, nil) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - nodeNames = tester.GetNodeNames() - gomega.Expect(nodeNames).NotTo(gomega.BeNil()) - ginkgo.By("Should not have tainted nodes") - defer results.RecordResult(identifiers.TestNonTaintedNodeKernelsIdentifier) - if len(nodeNames) == 0 { - ginkgo.Skip("Can't test tainted nodes when list of nodes is empty. Please check previous tests.") - } + var taintedNodes []string - for _, node := range nodeNames { + for _, node := range env.Nodes { context := common.GetContext() - tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node) + tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node.Name) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(testResult).NotTo(gomega.Equal(tnf.ERROR)) gomega.Expect(err).To(gomega.BeNil()) if testResult == tnf.FAILURE { - taintedNodes = append(taintedNodes, node) + taintedNodes = append(taintedNodes, node.Name) } } gomega.Expect(taintedNodes).To(gomega.BeNil()) }) } -func testHugepages() { - var nodeNames []string +func testHugepages(env *config.TestEnvironment) { var clusterHugepages, clusterHugepagesz int testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) - ginkgo.By("Should return list of worker node names") - context := common.GetContext() - tester := nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{"node-role.kubernetes.io/worker": nil}) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - nodeNames = tester.GetNodeNames() - gomega.Expect(nodeNames).NotTo(gomega.BeNil()) - ginkgo.By("Should return cluster's hugepages configuration") - context = common.GetContext() + context := common.GetContext() hugepageTester := hugepages.NewHugepages(common.DefaultTimeout) - test, err = tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) + test, err := tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err = test.Run() + testResult, err := test.Run() gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) clusterHugepages = hugepageTester.GetHugepages() @@ -355,16 +327,19 @@ func testHugepages() { ginkgo.By("Should have same configuration as cluster") ginkgo.By(fmt.Sprintf("cluster is configured with clusterHugepages=%d ; clusterHugepagesz=%d", clusterHugepages, clusterHugepagesz)) var badNodes []string - for _, node := range nodeNames { + for _, node := range env.Nodes { + if !node.IsWorker() { + continue + } context := common.GetContext() - tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node, clusterHugepagesz, clusterHugepages) + tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node.Name, clusterHugepagesz, clusterHugepages) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(err).To(gomega.BeNil()) if testResult != tnf.SUCCESS { - badNodes = append(badNodes, node) - ginkgo.By(fmt.Sprintf("node=%s hugepage config does not match machineconfig", node)) + badNodes = append(badNodes, node.Name) + ginkgo.By(fmt.Sprintf("node=%s hugepage config does not match machineconfig", node.Name)) } } gomega.Expect(badNodes).To(gomega.BeNil()) From 6b5cb84a44e01e67225355b1d7be12bc355e850b Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 6 Oct 2021 13:47:16 +0200 Subject: [PATCH 077/344] TNF handler generation tool improvements. (#378) * TNF handler generation tool improvements. Fixed so the tnf tool can be launched anywhere. If no envar TNF_HANDLER_SRC detected, the tool assumes you're in the repo root folder so the handlers folder should be at ./pkg/tnf/handlers. + Minor refactors. * Fixes proposed by hamadise. * Addressing David's comment. * Added link for handler templates. Co-authored-by: Salaheddine Hamadi --- DEVELOPING.md | 19 +++- cmd/tnf/main.go | 101 +++++++++++++----- .../handlers/handler_template/handler.tmpl | 38 ++++--- .../handler_template/handler_test.tmpl | 36 +++---- 4 files changed, 130 insertions(+), 64 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index fbb620119..ef5a74336 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -606,11 +606,22 @@ oc get pod %s -n %s -o go-template='{{len .spec.containers}}{{"\n"}}' ## Adding new handler -To facilitate adding new handlers, the developers can leverage existing infrastructure to generate repetitive code. -As an example, to generate handler with name Xyz, the command generate handler Xyz can be used(we need to run this command -in the folder "cmd/tnf/" and we run the command directly here). +To facilitate adding new handlers, the "tnf" utility has been created to help developers to avoid writing repetitive code. The tnf tool [source code is here](cmd/tnf) and can be built with the following command: +```shell-script +make build-tnf-tool +``` + +To generate a new handler named MyHandler, use the options "generate handler" as in the next example: +```shell-script +./tnf generate handler MyHandler +``` + The generated code has a template and creates the necessary headers. -The result is folder "new handler" located in /pkg/tnf/handlers/"new handler" that includes 3 files by handler template. +The result is folder "myhandler" located in /pkg/tnf/handlers/myhandler that includes 3 files by handler template. +The command relays on golang templates located in [pkg/tnf/handlers/handler_template](pkg/tnf/handlers/handler_template), so in case the "tnf" utility is executed outside the test-network-function root folder, the user can export the environment variable TNF_HANDLERS_SRC pointing to an existing "handlers" relative/absolute folder path. +```shell-script + export TNF_HANDLERS_SRC=other/path/pkg/tnf/handlers +``` ## Adding information to claim file diff --git a/cmd/tnf/main.go b/cmd/tnf/main.go index 3b914e8bf..a516119ee 100644 --- a/cmd/tnf/main.go +++ b/cmd/tnf/main.go @@ -2,12 +2,12 @@ package main import ( "bufio" - "log" "os" "path" "strings" "text/template" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -16,11 +16,13 @@ type myHandler struct { LowerHandlername string } -var ( - pathrelativetoroot string - handlerDirectory string - handlername string +const ( + envHandlersFolder = "TNF_HANDLERS_SRC" + docFileName = "doc.go" + handlerFolderPerms = 0755 +) +var ( rootCmd = &cobra.Command{ Use: "tnf", Short: "A CLI for creating, validating, and test-network-function tests.", @@ -36,53 +38,94 @@ var ( Short: "adding new handler.", RunE: generateHandlerFiles, } + + defaultHandlersFolder = path.Join("pkg", "tnf", "handlers") ) -func generateHandlerFiles(cmd *cobra.Command, args []string) error { - handlername = args[0] - pathrelativetoroot = path.Join("..", "..") - handlerDirectory = path.Join(pathrelativetoroot, "pkg", "tnf", "handlers") - newHandlerDirectory := path.Join(handlerDirectory, handlername) +func getHandlersDirectory() (string, error) { + handlersDirectory := os.Getenv(envHandlersFolder) - err := os.Mkdir(newHandlerDirectory, 0755) - if err != nil { - return err + if handlersDirectory == "" { + log.Warnf("Environment variable %s not set. Handlers base folder will be set to ./%s", + envHandlersFolder, defaultHandlersFolder) + + handlersDirectory = defaultHandlersFolder + } else { + log.Infof("Env var %s found. Handlers directory: %s", envHandlersFolder, handlersDirectory) } - myhandler := myHandler{LowerHandlername: handlername, UpperHandlername: strings.Title(handlername)} + // Convert to absolute path. + if !path.IsAbs(handlersDirectory) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + handlersDirectory = path.Join(cwd, handlersDirectory) + } - // pathfile this is the path of the file from template file that will creat + return handlersDirectory, nil +} - pathfile := path.Join(handlerDirectory, "handler_template", "doc.tmpl") - namefile := "" + "doc.go" - err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by doc.tmpl +func generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory string, myhandler myHandler) error { + type fileToRender struct { + templatePath string + renderedFileName string + } + + filesToRender := []fileToRender{ + {templatePath: path.Join(handlerTemplatesDirectory, "doc.tmpl"), renderedFileName: docFileName}, + {templatePath: path.Join(handlerTemplatesDirectory, "handler_test.tmpl"), renderedFileName: myhandler.LowerHandlername + "_test.go"}, + {templatePath: path.Join(handlerTemplatesDirectory, "handler.tmpl"), renderedFileName: myhandler.LowerHandlername + ".go"}, + } + + for _, renderedFileName := range filesToRender { + if err := createfile(renderedFileName.templatePath, renderedFileName.renderedFileName, myhandler, newHandlerDirectory); err != nil { + log.Errorf("Unable to create rendered file %s on %s", renderedFileName, newHandlerDirectory) + return err + } + } + + return nil +} + +func generateHandlerFiles(cmd *cobra.Command, args []string) error { + handlername := args[0] + myhandler := myHandler{LowerHandlername: strings.ToLower(handlername), UpperHandlername: strings.Title(handlername)} + + handlersDirectory, err := getHandlersDirectory() if err != nil { + log.Fatalf("Unable to get handlers path.") return err } - pathfile = path.Join(handlerDirectory, "handler_template", "handler_test.tmpl") - namefile = "" + myhandler.LowerHandlername + "_test.go" - err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template_test.tmpl + handlerTemplatesDirectory := path.Join(handlersDirectory, "handler_template") + + log.Infof("Using absolute path for tnf handlers directory: %s", handlersDirectory) + newHandlerDirectory := path.Join(handlersDirectory, myhandler.LowerHandlername) + + err = os.Mkdir(newHandlerDirectory, handlerFolderPerms) if err != nil { - return err + log.Fatal("Unable to create handler directory " + newHandlerDirectory) + os.Exit(1) } - pathfile = path.Join(handlerDirectory, "handler_template", "handler.tmpl") - namefile = "" + myhandler.LowerHandlername + ".go" - err = createfile(pathfile, namefile, myhandler, newHandlerDirectory) // here creating file by template.tmpl + err = generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory, myhandler) if err != nil { return err } - return err + + log.Infof("Handler files for %s successfully created in %s\n", myhandler.UpperHandlername, path.Join(newHandlerDirectory)) + return nil } -func createfile(pathfile, namefile string, myhandler myHandler, newHandlerDirectory string) error { - ftpl, err := template.ParseFiles(pathfile) +func createfile(templateFilePath, outputFileName string, myhandler myHandler, newHandlerDirectory string) error { + ftpl, err := template.ParseFiles(templateFilePath) if err != nil { return err } - temp := path.Join(newHandlerDirectory, namefile) + temp := path.Join(newHandlerDirectory, outputFileName) f, err := os.Create(temp) if err != nil { return err diff --git a/pkg/tnf/handlers/handler_template/handler.tmpl b/pkg/tnf/handlers/handler_template/handler.tmpl index 39f5e886c..9ae74ff9b 100644 --- a/pkg/tnf/handlers/handler_template/handler.tmpl +++ b/pkg/tnf/handlers/handler_template/handler.tmpl @@ -20,10 +20,11 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) -// {{ .UpperHandlername }} provides a {{ .UpperHandlername }} test implemented using command line tool {{ .UpperHandlername }}. +// {{ .UpperHandlername }} is the reel handler struct. type {{ .UpperHandlername }} struct { result int timeout time.Duration @@ -35,17 +36,28 @@ const ( // adding special variables ) -// Args function +// New{{ .UpperHandlername }} returns a new {{ .UpperHandlername }} handler struct. +// TODO: Add needed parameters to this function and initialize the handler properly. +func New{{ .UpperHandlername }}(timeout time.Duration) *{{ .UpperHandlername }} { + return &{{ .UpperHandlername }}{ + timeout: timeout, + result: tnf.ERROR, + args: []string{}, // TODO: Add proper execution command. + } +} + +// Args returns the initial execution/send command strings for handler {{ .UpperHandlername }}. func (h *{{ .UpperHandlername }}) Args() []string { return h.args } // GetIdentifier returns the tnf.Test specific identifier. -func (h *{{ .UpperHandlername }}) GetIdentifier() { - // Create identifier {{ .UpperHandlername }}Identifier. +func (h *{{ .UpperHandlername }}) GetIdentifier() identifier.Identifier { + // Return the {{ .UpperHandlername }} handler identifier. + return identifier.Identifier{} } -// Timeout return the timeout for the test. +// Timeout returns the timeout for the test. func (h *{{ .UpperHandlername }}) Timeout() time.Duration { return h.timeout } @@ -55,28 +67,28 @@ func (h *{{ .UpperHandlername }}) Result() int { return h.result } -// ReelFirst returns a step which expects an {{ .UpperHandlername }} summary for the given device. +// ReelFirst returns a reel step for handler {{ .UpperHandlername }}. func (h *{{ .UpperHandlername }}) ReelFirst() *reel.Step { return &reel.Step{ - Expect: []string{""}, // TODO : pass the list of possible regex in here + Expect: []string{}, // TODO : pass the list of possible regex in here Timeout: h.timeout, } } // ReelMatch parses the {{ .UpperHandlername }} output and set the test result on match. func (h *{{ .UpperHandlername }}) ReelMatch(_, _, match string) *reel.Step { + // TODO : add the matching logic here and return an appropriate tnf result. h.result = tnf.ERROR return nil - // TODO : add the matching logic in here } -// ReelTimeout does nothing, {{ .UpperHandlername }} requires no explicit intervention for a timeout. +// ReelTimeout function for {{ .UpperHandlername }} will be called by the reel FSM when a expect timeout occurs. func (h *{{ .UpperHandlername }}) ReelTimeout() *reel.Step { + // TODO : Add code here in case a timeout reaction is needed. return nil - // TODO : fill the stub } -// ReelEOF does nothing, {{ .UpperHandlername }} requires no explicit intervention for EOF. +// ReelEOF function for {{ .UpperHandlername }} will be called by the reel FSM when a EOF is read. func (h *{{ .UpperHandlername }}) ReelEOF() { - // TODO : fill the stub -} \ No newline at end of file + // TODO : Add code here in case a EOF reaction is needed. +} diff --git a/pkg/tnf/handlers/handler_template/handler_test.tmpl b/pkg/tnf/handlers/handler_template/handler_test.tmpl index 4f4ef5030..16263e3a6 100644 --- a/pkg/tnf/handlers/handler_template/handler_test.tmpl +++ b/pkg/tnf/handlers/handler_template/handler_test.tmpl @@ -24,37 +24,37 @@ const ( // adding special variable ) -// {{ .UpperHandlername }}_Args() +// Test_New{{ .UpperHandlername }} is the unit test for New{{ .UpperHandlername }}(). +func Test_New{{ .UpperHandlername }}(t *testing.T) { + // Todo: Write test. +} + +// Test_{{ .UpperHandlername }}_Args is the unit test for {{ .UpperHandlername }}_Args(). func Test{{ .UpperHandlername }}_Args(t *testing.T) { - // Todo: Write test for function {{ .UpperHandlername }}_Args. + // Todo: Write test. } -// {{ .UpperHandlername }}_GetIdentifier +// Test_{{ .UpperHandlername }}_GetIdentifier is the unit test for {{ .UpperHandlername }}_GetIdentifier(). func Test{{ .UpperHandlername }}_GetIdentifier(t *testing.T) { - // Todo : write test for function {{ .UpperHandlername }}_GetIdentifier. + // Todo: Write test. } -// {{ .UpperHandlername }}_ReelFirst +// Test_{{ .UpperHandlername }}_ReelFirst is the unit test for {{ .UpperHandlername }}_ReelFirst(). func Test{{ .UpperHandlername }}_ReelFirst(t *testing.T) { - // Todo : write test for function {{ .UpperHandlername }}_ReelFirst. + // Todo: Write test. } -// {{ .UpperHandlername }}_ReelEof -func Test{{ .UpperHandlername }}_ReelEof(t *testing.T) { - // Todo : write test for function {{ .UpperHandlername }}_ReelEof. +// Test_{{ .UpperHandlername }}_ReelEOF is the unit test for {{ .UpperHandlername }}_ReelEOF(). +func Test{{ .UpperHandlername }}_ReelEOF(t *testing.T) { + // Todo: Write test. } -// {{ .UpperHandlername }}_ReelTimeout +// Test_{{ .UpperHandlername }}_ReelTimeout is the unit test for {{ .UpperHandlername }}}_ReelTimeout(). func Test{{ .UpperHandlername }}_ReelTimeout(t *testing.T) { - // Todo : write test for function {{ .UpperHandlername }}_ReelTimeout. + // Todo: Write test. } -// {{ .UpperHandlername }}_ReelMatch +// Test_{{ .UpperHandlername }}_ReelMatch is the unit test for {{ .UpperHandlername }}_ReelMatch(). func Test{{ .UpperHandlername }}_ReelMatch(t *testing.T) { - // Todo : write test for function {{ .UpperHandlername }}_ReelMatch. -} - -// New{{ .UpperHandlername }} -func TestNew{{ .UpperHandlername }}(t *testing.T) { - // Todo : write test for function TestNew{{ .UpperHandlername }}. + // Todo: Write test. } From 1db83ed88f6d95dd79916169e5f3674458144120 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 7 Oct 2021 14:04:57 +0200 Subject: [PATCH 078/344] Fixed some linter warnings. (#381) --- cmd/oc/main.go | 10 ++++++++-- cmd/ping/main.go | 10 ++++++++-- cmd/ssh/main.go | 10 ++++++++-- test-network-function/diagnostic/suite.go | 6 ++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cmd/oc/main.go b/cmd/oc/main.go index ae3477cd0..4a39dfd3e 100644 --- a/cmd/oc/main.go +++ b/cmd/oc/main.go @@ -34,10 +34,16 @@ const ( // mandatoryNumArgs is the number of positional arguments required. mandatoryNumArgs = 3 + + // testTimeoutSecs timeout. + testTimeoutSecs = 2 + + // testPingCount number of ping packets to send. + testPingCount = 5 ) func parseArgs() (*interactive.Oc, <-chan error, string, time.Duration, error) { //nolint:gocritic //permit unnamed return values - timeout := flag.Int("t", 2, "Timeout in seconds") + timeout := flag.Int("t", testTimeoutSecs, "Timeout in seconds") flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [-t timeout] pod container targetIpAddress ?oc-exec-opt ... oc-exec-opt?\n", os.Args[0]) flag.PrintDefaults() @@ -70,7 +76,7 @@ func main() { os.Exit(tnf.ExitCodeMap[result]) } - request := ping.NewPing(timeoutDuration, targetIPAddress, 5) + request := ping.NewPing(timeoutDuration, targetIPAddress, testPingCount) chain := []reel.Handler{request} test, err := tnf.NewTest(oc.GetExpecter(), request, chain, ch) diff --git a/cmd/ping/main.go b/cmd/ping/main.go index 39d2eb8b6..02df9ed24 100644 --- a/cmd/ping/main.go +++ b/cmd/ping/main.go @@ -34,11 +34,17 @@ const ( // mandatoryNumArgs is the number of positional arguments required. mandatoryNumArgs = 1 + + // testTimeoutSecs timeout. + testTimeoutSecs = 2 + + // testNumRequests number of requests to send. + testNumRequests = 1 ) func parseArgs() (*ping.Ping, time.Duration) { - timeout := flag.Int("t", 2, "Timeout in seconds") - count := flag.Int("c", 1, "Number of requests to send") + timeout := flag.Int("t", testTimeoutSecs, "Timeout in seconds") + count := flag.Int("c", testNumRequests, "Number of requests to send") flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [-t timeout] [-c count] host\n", os.Args[0]) flag.PrintDefaults() diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index c9be31e2e..3db45001f 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -34,10 +34,16 @@ const ( // mandatoryNumArgs is the number of positional arguments required. mandatoryNumArgs = 3 + + // testTimeoutSecs timeout. + testTimeoutSecs = 2 + + // testPingCount number of ping packets to send. + testPingCount = 5 ) func parseArgs() (*interactive.Context, string, time.Duration, error) { - timeout := flag.Int("t", 2, "Timeout in seconds") + timeout := flag.Int("t", testTimeoutSecs, "Timeout in seconds") flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [-t timeout] user host targetIpAddress\n", os.Args[0]) flag.PrintDefaults() @@ -67,7 +73,7 @@ func main() { os.Exit(tnf.ExitCodeMap[result]) } - request := ping.NewPing(timeoutDuration, targetIPAddress, 5) + request := ping.NewPing(timeoutDuration, targetIPAddress, testPingCount) chain := []reel.Handler{request} test, err := tnf.NewTest(context.GetExpecter(), request, chain, context.GetErrorChannel()) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 6d40263a4..ebaa38ca1 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -257,6 +257,7 @@ func testNodesHwInfo() { func getNodeLscpu(nodeName string) map[string]string { const command = "lscpu" + const numSplitSubstrings = 2 result := map[string]string{} context := common.GetContext() tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) @@ -266,7 +267,7 @@ func getNodeLscpu(nodeName string) map[string]string { gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) for _, line := range tester.Processed { - fields := strings.SplitN(line, ":", 2) + fields := strings.SplitN(line, ":", numSplitSubstrings) result[fields[0]] = strings.TrimSpace(fields[1]) } return result @@ -274,6 +275,7 @@ func getNodeLscpu(nodeName string) map[string]string { func getNodeIfconfig(nodeName string) map[string][]string { const command = "ifconfig" + const numSplitSubstrings = 2 result := map[string][]string{} context := common.GetContext() tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) @@ -288,7 +290,7 @@ func getNodeIfconfig(nodeName string) map[string][]string { continue } if line[0] != ' ' { - fields := strings.SplitN(line, ":", 2) + fields := strings.SplitN(line, ":", numSplitSubstrings) deviceName = fields[0] line = fields[1] } From 53ddeba4bf9f2893253817e0d91dcfc645c06490 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Thu, 7 Oct 2021 09:42:15 -0500 Subject: [PATCH 079/344] GolangCI-Lint 1.42.1; remove deprecated linters (#383) --- .github/workflows/merge.yaml | 4 ++-- .github/workflows/pre-main.yaml | 17 +++++++---------- .golangci.yml | 4 +--- Dockerfile | 2 +- Makefile | 2 -- README.md | 2 +- pkg/tnf/handlers/generic/generic_test.go | 4 ++-- pkg/tnf/test.go | 2 +- 8 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml index 4f8dd4bd1..f7c507dcf 100644 --- a/.github/workflows/merge.yaml +++ b/.github/workflows/merge.yaml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v2 with: - go-version: ^1.15 + go-version: ^1.16 - name: Check out code into the Go module directory uses: actions/checkout@v2 diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 38ffbf1f4..87de35b7b 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -29,10 +29,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v2 with: - go-version: ^1.15 + go-version: ^1.16 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -42,13 +42,10 @@ jobs: - name: Rebuild mocks run: go get github.com/golang/mock/mockgen && make mocks - - name: Install golint - run: go get golang.org/x/lint/golint - # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However # there does not appear to be a way to honor our existing .golangci.yml. For now, mimic developer behavior. - name: Install golangci-lint - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.39.0 + run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1 - name: make lint run: make lint @@ -58,10 +55,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v2 with: - go-version: ^1.15 + go-version: ^1.16 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -82,10 +79,10 @@ jobs: KUBECONFIG: '/home/runner/.kube/config' steps: - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v2 with: - go-version: ^1.15 + go-version: ^1.16 - name: Check out code into the Go module directory uses: actions/checkout@v2 diff --git a/.golangci.yml b/.golangci.yml index 44b2bea50..ccfd0cb40 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -28,8 +28,6 @@ linters-settings: min-complexity: 15 goimports: local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 gomnd: settings: mnd: @@ -72,7 +70,6 @@ linters: - gocyclo - gofmt - goimports - - golint - gomnd - goprintffuncname - gosec @@ -84,6 +81,7 @@ linters: - nakedret - noctx - nolintlint + - revive - rowserrcheck - staticcheck - stylecheck diff --git a/Dockerfile b/Dockerfile index 436206437..7ba566604 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG TNF_PARTNER_DIR=/usr/tnf-partner ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src -ENV GOLANGCI_VERSION=v1.32.2 +ENV GOLANGCI_VERSION=v1.42.1 ENV OPENSHIFT_VERSION=4.6.32 ENV TNF_DIR=/usr/tnf diff --git a/Makefile b/Makefile index 794d2e009..979e090b1 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,6 @@ clean: # Run configured linters lint: - golint -set_exit_status `go list ./... | grep -v vendor` golangci-lint run # Build and run unit tests @@ -146,5 +145,4 @@ update-deps: install-tools: go get github.com/onsi/ginkgo/ginkgo go get github.com/onsi/gomega/... - go get golang.org/x/lint/golint go get github.com/golang/mock/mockgen diff --git a/README.md b/README.md index b955691f0..b9e6965b5 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,7 @@ At a minimum, the following dependencies must be installed *prior* to running `m Dependency|Minimum Version ---|--- [GoLang](https://golang.org/dl/)|1.16 -[golangci-lint](https://golangci-lint.run/usage/install/)|1.32.2 +[golangci-lint](https://golangci-lint.run/usage/install/)|1.42.1 [jq](https://stedolan.github.io/jq/)|1.6 [OpenShift Client](https://docs.openshift.com/container-platform/4.4/welcome/index.html)|4.4 diff --git a/pkg/tnf/handlers/generic/generic_test.go b/pkg/tnf/handlers/generic/generic_test.go index 417ff5ab9..9e083d375 100644 --- a/pkg/tnf/handlers/generic/generic_test.go +++ b/pkg/tnf/handlers/generic/generic_test.go @@ -225,8 +225,8 @@ func TestGeneric(t *testing.T) { // this assertion also prevents `tester` from being `nil` inside the following `if` assert.Equal(t, testCase.expectedCreationErr, err != nil) if !testCase.expectedCreationErr { - assert.Equal(t, testCase.expectedTester, tester != nil) //nolint:staticcheck - assert.Equal(t, testCase.expectedTimeout, (*tester).Timeout()) //nolint:staticcheck + assert.Equal(t, testCase.expectedTester, tester != nil) + assert.Equal(t, testCase.expectedTimeout, (*tester).Timeout()) assert.Equal(t, testCase.expectedHandlers, handlers != nil) if testCase.expectedHandlers { diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index df9b76f60..fec844163 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -37,7 +37,7 @@ const ( // TestsExtraInfo a collection of messages per test that is added to the claim file // use WriteTestExtraInfo for writing to it -var TestsExtraInfo []map[string][]string = []map[string][]string{} +var TestsExtraInfo = []map[string][]string{} // CreateTestExtraInfoWriter creates a function that writes info messages for a specific test // info messages that were already added by calling the function will exist in the claim file From f1f2d6463b50649b36504eecb58c4a06ae26e7ac Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 7 Oct 2021 10:23:27 -0500 Subject: [PATCH 080/344] Adding Support for exit code in goexpect/commands and minor typo fixes to catalog (#377) * Adding Support for exit code in goexpect/commands * PR review * re-add support for unit tests * Jun PR review Co-authored-by: Jun Chen --- CATALOG.md | 4 +- pkg/tnf/identifier/identifiers.go | 2 +- pkg/tnf/reel/reel.go | 80 ++++++++++++++----- pkg/tnf/test.go | 6 ++ test-network-function/diagnostic/suite.go | 2 +- .../identifiers/identifiers.go | 2 +- 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 89bfed08c..12cbfb904 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -67,7 +67,7 @@ Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Ce Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/diagnostic/clusterversion extracts OCP versions from the cluster. +Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| ### http://test-network-function.com/testcases/diagnostic/extract-node-information @@ -239,7 +239,7 @@ A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the b Property|Description ---|--- Version|v1.0.0 -Description|extracts OCP versions from the cluster +Description|Extracts OCP versions from the cluster Result Type|normative Intrusive|false Modifications Persist After Test|false diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 4cbdfeba3..7b17263fd 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -599,7 +599,7 @@ var Catalog = map[string]TestCatalogEntry{ }, clusterVersionIdentifierURL: { Identifier: ClusterVersionIdentifier, - Description: "extracts OCP versions from the cluster", + Description: "Extracts OCP versions from the cluster", Type: Normative, IntrusionSettings: IntrusionSettings{ ModifiesSystem: false, diff --git a/pkg/tnf/reel/reel.go b/pkg/tnf/reel/reel.go index 42731033a..3ebd89368 100644 --- a/pkg/tnf/reel/reel.go +++ b/pkg/tnf/reel/reel.go @@ -19,9 +19,12 @@ package reel import ( "fmt" "regexp" + "strconv" "strings" "time" + log "github.com/sirupsen/logrus" + expect "github.com/google/goexpect" "google.golang.org/grpc/codes" ) @@ -29,14 +32,21 @@ import ( const ( // EndOfTestSentinel is the emulated terminal prompt that will follow command output. EndOfTestSentinel = `END_OF_TEST_SENTINEL` + // ExitKeyword keyword delimiting the command exit status + ExitKeyword = "exit=" ) var ( - // endOfTestSentinelCutset is used to trim a match of the EndOfTestSentinel. - endOfTestSentinelCutset = fmt.Sprintf("%s\n", EndOfTestSentinel) - // EndOfTestRegexPostfix is the postfix added to regular expressions to match the emulated terminal prompt - // (EndOfTestSentinel) - EndOfTestRegexPostfix = fmt.Sprintf("((.|\n)*%s\n)", EndOfTestSentinel) + + // matchSentinel This regular expression is matching stricly the sentinel and exit code. + // This match regular expression matches commands that return no output + matchSentinel = fmt.Sprintf("((.|\n)*%s %s[0-9]+\n)", EndOfTestSentinel, ExitKeyword) + + // EndOfTestRegexPostfix This regular expression is a postfix added to the goexpect regular expressions. This regular expression matches a + // sentinel or marker string that is marking the end of the command output. This is because after the command + // output, the shell might also return a prompt which is not desired. Note: this is currently the same as the string above + // but was splitted for clarity + EndOfTestRegexPostfix = matchSentinel ) // Step is an instruction for a single REEL pass. @@ -98,6 +108,8 @@ type Reel struct { } // DisableTerminalPromptEmulation disables terminal prompt emulation for the reel.Reel. +// This disables terminal shell management and is used only for unit testing where the terminal is not available +// In this mode, go expect operates only on strings not command/shell outputs func DisableTerminalPromptEmulation() Option { return func(r *Reel) Option { r.disableTerminalPromptEmulation = true @@ -109,6 +121,8 @@ func DisableTerminalPromptEmulation() Option { // array of strings is turned into a corresponding array of expect.Batcher. This method side-effects the input // expectations array, following the Builder design pattern. Finally, the first match is stored in the firstMatch // output parameter. +// This command translates individual expectations in the test cases (e.g. success, failure, etc) into expect.Case in go expect +// The expect.Case are later matched in order inside goexpect ExpectBatch function. func (r *Reel) batchExpectations(expectations []string, batcher []expect.Batcher, firstMatch *string) []expect.Batcher { if len(expectations) > 0 { expectCases := r.generateCases(expectations, firstMatch) @@ -123,10 +137,15 @@ func (r *Reel) batchExpectations(expectations []string, batcher []expect.Batcher // parameter to store the first match found in the expectations array. Thus, the order of expectations is important. func (r *Reel) generateCases(expectations []string, firstMatch *string) []expect.Caser { var cases []expect.Caser + // expectations created from test case matches for _, expectation := range expectations { thisCase := r.generateCase(expectation, firstMatch) cases = append(cases, thisCase) } + // extra test case to match when commands do not return anything but exit without error. This expectation makes + // sure that any command exiting successfully will be processed without timeout. + thisCase := r.generateCase("", firstMatch) + cases = append(cases, thisCase) return cases } @@ -142,7 +161,7 @@ func (r *Reel) generateCase(expectation string, firstMatch *string) *expect.Case }} } -// Each Step can have exactly one execution string (Step.Execute). This method follows the Adapter design pattern; a +// Each Step can have exactly one execution string (Step.Execute). This method follows the Adapter design pattern; a // single raw execution string is converted into a corresponding expect.Batcher. The function returns an array of // expect.Batcher, as it is expected that there are likely expectations to follow. func (r *Reel) generateBatcher(execute string) []expect.Batcher { @@ -170,8 +189,9 @@ func (r *Reel) Step(step *Step, handler Handler) error { exec, exp, timeout := step.unpack() var batcher []expect.Batcher batcher = r.generateBatcher(exec) - var firstMatch string - batcher = r.batchExpectations(exp, batcher, &firstMatch) + // firstMatchRe is the first regular expression (expectation) that has matched results + var firstMatchRe string + batcher = r.batchExpectations(exp, batcher, &firstMatchRe) results, err := (*r.expecter).ExpectBatch(batcher, timeout) if !step.hasExpectations() { @@ -188,9 +208,12 @@ func (r *Reel) Step(step *Step, handler Handler) error { if len(results) > 0 { result := results[0] - output := r.stripEmulatedPromptFromOutput(result.Output) - match := r.stripEmulatedPromptFromOutput(result.Match[0]) - + output, outputStatus := r.stripEmulatedPromptFromOutput(result.Output) + if outputStatus != 0 { + r.Err = fmt.Errorf("error executing command %d: ", outputStatus) + } + match, matchStatus := r.stripEmulatedPromptFromOutput(result.Match[0]) + log.Infof("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d", output, match, outputStatus, matchStatus) matchIndex := strings.Index(output, match) var before string // special case: the match regex may be nothing at all. @@ -199,7 +222,8 @@ func (r *Reel) Step(step *Step, handler Handler) error { } else { before = "" } - step = handler.ReelMatch(r.stripEmulatedRegularExpression(firstMatch), before, match) + strippedFirstMatchRe := r.stripEmulatedRegularExpression(firstMatchRe) + step = handler.ReelMatch(strippedFirstMatchRe, before, match) } } } @@ -254,15 +278,34 @@ func (r *Reel) wrapTestCommand(cmd string) string { // WrapTestCommand wraps cmd so that the output will end in an emulated terminal prompt. func WrapTestCommand(cmd string) string { cmd = strings.TrimRight(cmd, "\n") - return fmt.Sprintf("%s && echo %s\n", cmd, EndOfTestSentinel) + wrappedCommand := fmt.Sprintf("%s ; echo %s %s$?\n", cmd, EndOfTestSentinel, ExitKeyword) + log.Tracef("Command sent: %s", wrappedCommand) + return wrappedCommand } // stripEmulatedPromptFromOutput will elide the emulated terminal prompt from the test output. -func (r *Reel) stripEmulatedPromptFromOutput(output string) string { - if !r.disableTerminalPromptEmulation { - return strings.TrimRight(strings.TrimRight(output, endOfTestSentinelCutset), "\n") +func (r *Reel) stripEmulatedPromptFromOutput(output string) (data string, status int) { + parsed := strings.Split(output, EndOfTestSentinel) + var err error + if !r.disableTerminalPromptEmulation && len(parsed) == 2 { + // if a sentinel was present, then we have at least 2 parsed results + // if command retuned nothing parsed[0]=="" + data = parsed[0] + status, err = strconv.Atoi(strings.Split(strings.Split(parsed[1], ExitKeyword)[1], "\n")[0]) + if err != nil { + // Cannot parse status from output, something is wrong, fail command + status = 1 + log.Errorf("Cannot determine command status. Error: %s", err) + } + // remove trailing \n if present + data = strings.TrimRight(data, "\n") + } else { + // to support unit tests (without sentinel parsing) + data = output + status = 0 + log.Errorf("Cannot determine command status, no sentinel present. Error: %s", err) } - return output + return } // stripEmulatedRegularExpression will elide the modified part of the terminal prompt regular expression. @@ -276,7 +319,8 @@ func (r *Reel) stripEmulatedRegularExpression(match string) string { // addEmulatedRegularExpression will append the additional regular expression to capture the emulated terminal prompt. func (r *Reel) addEmulatedRegularExpression(regularExpressionString string) string { if !r.disableTerminalPromptEmulation { - return fmt.Sprintf("%s%s", regularExpressionString, EndOfTestRegexPostfix) + regularExpressionStringWithMetadata := fmt.Sprintf("%s%s", regularExpressionString, EndOfTestRegexPostfix) + return regularExpressionStringWithMetadata } return regularExpressionString } diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index fec844163..c09fd4501 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -19,6 +19,8 @@ package tnf import ( "time" + log "github.com/sirupsen/logrus" + expect "github.com/google/goexpect" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -85,6 +87,10 @@ type Test struct { // Run performs a test, returning the result and any encountered errors. func (t *Test) Run() (int, error) { err := t.runner.Run(t) + // if the runner fails, print the error + if t.runner.Err != nil { + log.Errorf("%s", t.runner.Err) + } return t.tester.Result(), err } diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index ebaa38ca1..5f12dcf84 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -216,8 +216,8 @@ func testOcpVersion() { test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) versionsOcp = tester.GetVersions() } diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 651079b2c..5ad787235 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -528,6 +528,6 @@ the changes for you.`, Identifier: TestclusterVersionIdentifier, Type: informativeResult, Description: formDescription(TestclusterVersionIdentifier, - `extracts OCP versions from the cluster.`), + `Extracts OCP versions from the cluster.`), }, } From 82fef44fc7956e51ff24bafc1c4993171bf74505 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 7 Oct 2021 11:57:32 -0400 Subject: [PATCH 081/344] README update for session timeout requirement (#376) * README update for session timeout requirement * Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b9e6965b5..a73a228ee 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,8 @@ The image can be pulled using : docker pull quay.io/testnetworkfunction/test-network-function ``` ### Cluster requirement -To run all the tests, OCP cluster should provide enough resources to drain nodes and reschedule pods. If that's not the case, then ``lifecycle-pod-recreation`` test should be skipped. +* OCP cluster should allow interactive shell sessions to pods/containers to stay alive when being idle for more than a few minutes. If it is not the case, consult the maintainer of the cluster infrastructure on how it can be enabled. Also, make sure the firewalls/load balancers on the path do not timeout idle connections too quickly. +* OCP cluster should provide enough resources to drain nodes and reschedule pods. If that's not the case, then ``lifecycle-pod-recreation`` test should be skipped. ### Check cluster resources Some tests suites such as platform-alteration require node access to get node configuration like hugepage. In order to get the required information, the test suite does not ssh into nodes, but instead rely on [oc debug tools ](https://docs.openshift.com/container-platform/3.7/cli_reference/basic_cli_operations.html#debug). This tool makes it easier to fetch information from nodes and also to debug running pods. From 81dc29f875c0ad5d9f62cb03471e05334ea8d449 Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Thu, 7 Oct 2021 19:31:56 +0300 Subject: [PATCH 082/344] =?UTF-8?q?CNFCERT-148:Update=20test=20suite=20to?= =?UTF-8?q?=20support=20statefulset=20and=20stateful=20app=E2=80=A6=20(#38?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CNFCERT-148:Update test suite to support statefulset and stateful applications * CNFCERT-148:Update identefires * CNFCERT-148:Update the consts * CNFCERT-148:Update the consts * CNFCERT-148:Update the consts Co-authored-by: Jun Chen --- CATALOG.md | 6 +++--- pkg/tnf/handlers/owners/owners.go | 9 ++++++++- pkg/tnf/handlers/owners/owners_test.go | 1 + pkg/tnf/identifier/identifiers.go | 2 +- test-network-function/identifiers/identifiers.go | 4 ++-- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 12cbfb904..c3f6a197f 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -115,9 +115,9 @@ Suggested Remediation|In high availability cases, Pod podAntiAffinity rule shoul Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s). +Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s). Result Type|normative -Suggested Remediation|Deploy the CNF using ReplicaSet. +Suggested Remediation|Deploy the CNF using ReplicaSet/StatefulSet. ### http://test-network-function.com/testcases/lifecycle/pod-recreation Property|Description @@ -529,7 +529,7 @@ Runtime Binaries Required|`oc` Property|Description ---|--- Version|v1.0.0 -Description|A generic test used to verify pod is managed by a ReplicaSet +Description|A generic test used to verify pod is managed by a ReplicaSet/StatefulSet Result Type|normative Intrusive|false Modifications Persist After Test|false diff --git a/pkg/tnf/handlers/owners/owners.go b/pkg/tnf/handlers/owners/owners.go index 882b68eb1..e1508d6cf 100644 --- a/pkg/tnf/handlers/owners/owners.go +++ b/pkg/tnf/handlers/owners/owners.go @@ -27,6 +27,12 @@ import ( const ( owRegex = "(?s)OWNERKIND\n.+" + // statefulSet variable + statefulSet = "StatefulSet" + // replicaSet variable + replicaSet = "ReplicaSet" + // daemonSet variable + daemonSet = "DaemonSet" ) // Owners tests pod owners @@ -76,7 +82,8 @@ func (ow *Owners) ReelFirst() *reel.Step { // ReelMatch ensures that list of nodes is not empty and stores the names as []string func (ow *Owners) ReelMatch(_, _, match string) *reel.Step { - if strings.Contains(match, "ReplicaSet") && !strings.Contains(match, "DaemonSet") { + if (strings.Contains(match, statefulSet) || strings.Contains(match, replicaSet)) && + !strings.Contains(match, daemonSet) { ow.result = tnf.SUCCESS } else { ow.result = tnf.FAILURE diff --git a/pkg/tnf/handlers/owners/owners_test.go b/pkg/tnf/handlers/owners/owners_test.go index 52f3dcf51..4f5b31153 100644 --- a/pkg/tnf/handlers/owners/owners_test.go +++ b/pkg/tnf/handlers/owners/owners_test.go @@ -95,6 +95,7 @@ var ( } testInputSuccessSlice = []string{ "OWNERKIND\nReplicaSet\n", + "OWNERKIND\nStatefulSet\n", "OWNERKIND\nOwner\nOWNERKIND\nReplicaSet\n", } ) diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 7b17263fd..340e2f673 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -423,7 +423,7 @@ var Catalog = map[string]TestCatalogEntry{ }, ownersIdentifierURL: { Identifier: OwnersIdentifier, - Description: "A generic test used to verify pod is managed by a ReplicaSet", + Description: "A generic test used to verify pod is managed by a ReplicaSet/StatefulSet", Type: Normative, IntrusionSettings: IntrusionSettings{ ModifiesSystem: false, diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 5ad787235..db8aa8e80 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -380,9 +380,9 @@ ClusterRoleBindings, if possible.`, TestPodDeploymentBestPracticesIdentifier: { Identifier: TestPodDeploymentBestPracticesIdentifier, Type: normativeResult, - Remediation: `Deploy the CNF using ReplicaSet.`, + Remediation: `Deploy the CNF using ReplicaSet/StatefulSet.`, Description: formDescription(TestPodDeploymentBestPracticesIdentifier, - `tests that CNF Pod(s) are deployed as part of a ReplicaSet(s).`), + `tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s).`), }, TestPodRoleBindingsBestPracticesIdentifier: { From 1e922d1de1e13a9d0bc653b8cff4fd9ed087ff7f Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 8 Oct 2021 15:39:04 +0200 Subject: [PATCH 083/344] Error log trace in SetLogFormat() set to Info. (#387) --- test-network-function/common/env.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index be27813af..456583910 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -112,5 +112,5 @@ func SetLogFormat() { return strconv.Itoa(f.Line) + "]", fmt.Sprintf("[%s:", filename) } log.SetFormatter(customFormatter) - log.Error("debug format initialization: done") + log.Info("debug format initialization: done") } From 7c2b1a58307753a7860651d62eabe3a75c395450 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 8 Oct 2021 16:02:41 +0200 Subject: [PATCH 084/344] Log traces improvements for shell spawning. (#386) - Added debug trace when spawning shells. - Added the name and the shell cmd in the mirroring pipes failure message. The goal is to ease the process of tracking the lifecycle of the spawned shell sessions when they crash. This is specially important when reading/interpreting big logs. Co-authored-by: Jun Chen --- pkg/tnf/interactive/spawner.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/tnf/interactive/spawner.go b/pkg/tnf/interactive/spawner.go index b810d6175..c1dddcbf3 100644 --- a/pkg/tnf/interactive/spawner.go +++ b/pkg/tnf/interactive/spawner.go @@ -299,7 +299,7 @@ func logCmdMirrorPipe(cmdLine string, pipeToMirror io.Reader, name string, trace } if err != nil { // Some Error has happened, goroutine about to exit - log.Warnf("Exiting cmd log mirroring goroutine. Error: %s", err) + log.Warnf("Exiting %s log mirroring goroutine for cmd %s. Error: %s", name, cmdLine, err) return } } @@ -327,6 +327,8 @@ func (g *GoExpectSpawner) Spawn(command string, args []string, timeout time.Dura } cmdLine := fmt.Sprintf("%s %s", command, strings.Join(args, " ")) + log.Debugf("Spawning interactive shell. Cmd: %s", cmdLine) + logCmdMirrorPipe(cmdLine, stderrPipe, "STDERR", false) stdoutPipe = logCmdMirrorPipe(cmdLine, stdoutPipe, "STDOUT", true) @@ -355,7 +357,7 @@ func (g *GoExpectSpawner) spawnGeneric(spawnFunc *SpawnFunc, stdinPipe io.WriteC }, Check: func() bool { if !(*spawnFunc).IsRunning() { - log.Error("Unable to spawn shell. Cmd: " + strings.Join((*spawnFunc).Args(), " ")) + log.Error("Unable to send commands to spawned shell. Shell cmd: " + strings.Join((*spawnFunc).Args(), " ")) return false } return true From 81a2ab62202c060d8cac48e718929bc244eedbff Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 8 Oct 2021 16:14:58 +0200 Subject: [PATCH 085/344] Fixed hugepages TC of the platform-alteration TS. (#384) The way to get the cluster machine config was wrong, because it was not selecting the correct one: the master/worker-rendered objects usually don't have the worker/role label. Co-authored-by: Jun Chen --- pkg/tnf/handlers/hugepages/hugepages.go | 23 +++++++----- pkg/tnf/handlers/hugepages/hugepages_test.go | 15 ++++---- test-network-function/platform/suite.go | 37 +++++++++++--------- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/pkg/tnf/handlers/hugepages/hugepages.go b/pkg/tnf/handlers/hugepages/hugepages.go index c905f8cca..7cce08bac 100644 --- a/pkg/tnf/handlers/hugepages/hugepages.go +++ b/pkg/tnf/handlers/hugepages/hugepages.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" @@ -37,23 +38,25 @@ const ( // Hugepages holds information derived from running "oc get MachineConfig" on the command line. type Hugepages struct { - hugepagesz int - hugepages int - result int - timeout time.Duration - args []string + hugepagesz int + hugepages int + result int + timeout time.Duration + args []string + machineConfig string } // NewHugepages creates a new Hugepages tnf.Test. -func NewHugepages(timeout time.Duration) *Hugepages { +func NewHugepages(timeout time.Duration, machineConfig string) *Hugepages { return &Hugepages{ timeout: timeout, result: tnf.ERROR, args: []string{ - "oc", "get", "machineconfigs", "-l", "machineconfiguration.openshift.io/role=worker", + "oc", "get", "machineconfig", machineConfig, "-o", "custom-columns=KARGS:.spec.kernelArguments", - "|", "grep", "-v", "nil", "|", "grep", "-E", "'hugepage|KARGS'", + "|", "grep", "-E", "'hugepage|KARGS'", }, + machineConfig: machineConfig, } } @@ -115,6 +118,8 @@ func (hp *Hugepages) ReelMatch(_, _, match string) *reel.Step { if ok { hp.hugepages, _ = strconv.Atoi(hugepages) } else { + logrus.Warnf("Hugepages param not found in machineconfig %s. Defaulting to %d.", + hp.machineConfig, RhelDefaultHugepages) hp.hugepages = RhelDefaultHugepages } @@ -126,6 +131,8 @@ func (hp *Hugepages) ReelMatch(_, _, match string) *reel.Step { if ok { hp.hugepagesz = atoi(hugepagesz) } else { + logrus.Warnf("Hugepagesz param not found in machineconfig %s. Defaulting to %d.", + hp.machineConfig, RhelDefaultHugepagesz) hp.hugepagesz = RhelDefaultHugepagesz } } diff --git a/pkg/tnf/handlers/hugepages/hugepages_test.go b/pkg/tnf/handlers/hugepages/hugepages_test.go index 01ef73a44..989b487e8 100644 --- a/pkg/tnf/handlers/hugepages/hugepages_test.go +++ b/pkg/tnf/handlers/hugepages/hugepages_test.go @@ -27,14 +27,14 @@ import ( ) func Test_NewHugepages(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) assert.Equal(t, testTimeoutDuration, newHp.Timeout()) assert.Equal(t, newHp.Result(), tnf.ERROR) } func Test_ReelFirstPositive(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) firstStep := newHp.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -44,7 +44,7 @@ func Test_ReelFirstPositive(t *testing.T) { } func Test_ReelFirstPositiveEmpty(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) firstStep := newHp.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -54,7 +54,7 @@ func Test_ReelFirstPositiveEmpty(t *testing.T) { } func Test_ReelFirstNegative(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) firstStep := newHp.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -63,7 +63,7 @@ func Test_ReelFirstNegative(t *testing.T) { } func Test_ReelMatchSuccessEmpty(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) step := newHp.ReelMatch("", "", testInputEmpty) assert.Nil(t, step) @@ -73,7 +73,7 @@ func Test_ReelMatchSuccessEmpty(t *testing.T) { } func Test_ReelMatchSuccess(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) step := newHp.ReelMatch("", "", testInputSuccess) assert.Nil(t, step) @@ -84,13 +84,14 @@ func Test_ReelMatchSuccess(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newHp := hp.NewHugepages(testTimeoutDuration) + newHp := hp.NewHugepages(testTimeoutDuration, testMachineConfig) assert.NotNil(t, newHp) newHp.ReelEOF() } const ( testTimeoutDuration = time.Second * 2 + testMachineConfig = "worker-rendered-1" testInputError = "" testInputEmpty = "KARGS\n" testInputSuccess = "KARGS\n[skew_tick=1 nohz=on rcu_nocbs=2-19,22-39,42-59,62-79 tuned.non_isolcpus=30000300,00300003 intel_pstate=disable nosoftlockup tsc=nowatchdog intel_iommu=on iommu=pt isolcpus=managed_irq,2-19,22-39,42-59,62-79 systemd.cpu_affinity=0,1,40,41,20,21,60,61 hugepages=32 default_hugepagesz=32M hugepages=64 hugepagesz=1G nmi_watchdog=0 audit=0 mce=off processor.max_cstate=1 idle=poll intel_idle.max_cstate=0]\n" diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 28708a1fc..a76f24b1f 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -308,38 +308,41 @@ func testTainted(env *config.TestEnvironment) { }) } +func getNodeMcHugepages(nodeName string) (hugePagesCount, hugePagesSize int) { + context := common.GetContext() + mcName := getMcName(context, nodeName) + hugepageTester := hugepages.NewHugepages(common.DefaultTimeout, mcName) + test, err := tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + testResult, err := test.Run() + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(err).To(gomega.BeNil()) + return hugepageTester.GetHugepages(), hugepageTester.GetHugepagesz() +} + func testHugepages(env *config.TestEnvironment) { - var clusterHugepages, clusterHugepagesz int testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) - ginkgo.By("Should return cluster's hugepages configuration") - context := common.GetContext() - hugepageTester := hugepages.NewHugepages(common.DefaultTimeout) - test, err := tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) - clusterHugepages = hugepageTester.GetHugepages() - clusterHugepagesz = hugepageTester.GetHugepagesz() - - ginkgo.By("Should have same configuration as cluster") - ginkgo.By(fmt.Sprintf("cluster is configured with clusterHugepages=%d ; clusterHugepagesz=%d", clusterHugepages, clusterHugepagesz)) var badNodes []string + context := common.GetContext() for _, node := range env.Nodes { if !node.IsWorker() { continue } - context := common.GetContext() - tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node.Name, clusterHugepagesz, clusterHugepages) + ginkgo.By("Should return machineconfig hugepages configuration of node " + node.Name) + nodeHugePagesCount, nodeHugePagesSize := getNodeMcHugepages(node.Name) + + ginkgo.By(fmt.Sprintf("Node's machine config hugepages=%d/hugepagesz=%d values should match the actual ones in the node.", + nodeHugePagesCount, nodeHugePagesSize)) + tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node.Name, nodeHugePagesSize, nodeHugePagesCount) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) testResult, err := test.Run() gomega.Expect(err).To(gomega.BeNil()) if testResult != tnf.SUCCESS { badNodes = append(badNodes, node.Name) - ginkgo.By(fmt.Sprintf("node=%s hugepage config does not match machineconfig", node.Name)) + ginkgo.By(fmt.Sprintf("Node=%s hugepage config does not match machineconfig", node.Name)) } } gomega.Expect(badNodes).To(gomega.BeNil()) From ebcfcb8d5695c4035790afeff2e4aa32e2146fd6 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 8 Oct 2021 10:31:28 -0400 Subject: [PATCH 086/344] =?UTF-8?q?Streamlining=20the=20test=20invocation?= =?UTF-8?q?=20code=20with=20test.RunAndValidate()=20func=E2=80=A6=20(#385)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Streamlining the test invocation code with test.RunAndValidate() functions * Update suite.go --- pkg/tnf/test.go | 24 +++++++- test-network-function/accesscontrol/suite.go | 30 +++------- test-network-function/common/env.go | 8 --- test-network-function/diagnostic/suite.go | 50 ++++++---------- test-network-function/lifecycle/suite.go | 62 ++++++++------------ test-network-function/networking/suite.go | 6 +- test-network-function/observability/suite.go | 12 ++-- test-network-function/operator/suite.go | 16 ++--- test-network-function/platform/suite.go | 46 ++++++--------- 9 files changed, 102 insertions(+), 152 deletions(-) diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index c09fd4501..843449e41 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -135,13 +135,33 @@ func (t *Test) ReelEOF() { } } -// RunAndValidateTest runs the test and checks the result -func (t *Test) RunAndValidateTest() { +// RunAndValidate runs the test and checks the result +func (t *Test) RunAndValidate() { + t.RunAndValidateWithFailureCallback(nil) +} + +// RunAndValidateWithFailureCallback runs the test, checks the result/error and invokes the cb on failure +func (t *Test) RunAndValidateWithFailureCallback(cb func()) { testResult, err := t.Run() + if testResult == FAILURE && cb != nil { + cb() + } gomega.Expect(testResult).To(gomega.Equal(SUCCESS)) gomega.Expect(err).To(gomega.BeNil()) } +// RunWithFailureCallback runs the test, invokes the cb on failure +// This is useful when the testcase needs to continue whether this test result is success or not +func (t *Test) RunWithFailureCallback(cb func()) { + testResult, err := t.Run() + if testResult == FAILURE && cb != nil { + cb() + } + if err != nil { + log.Warnf("Test %s error: %v", t.tester.GetIdentifier().URL, err) + } +} + // NewTest creates a new Test given a chain of Handlers. func NewTest(expecter *expect.Expecter, tester Tester, chain []reel.Handler, errorChannel <-chan error, opts ...reel.Option) (*Test, error) { args := tester.Args() diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 800af5ff0..817dc51cb 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -113,9 +113,7 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + test.RunAndValidate() count++ } } else { @@ -125,9 +123,7 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + test.RunAndValidate() } } }) @@ -168,15 +164,14 @@ func testServiceAccount(env *config.TestEnvironment) { tester := serviceaccount.NewServiceAccount(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() serviceAccountName := tester.GetServiceAccountName() gomega.Expect(serviceAccountName).ToNot(gomega.BeEmpty()) } }) } +//nolint:dupl func testRoleBindings(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRoleBindingsBestPracticesIdentifier) ginkgo.It(testID, func() { @@ -194,18 +189,14 @@ func testRoleBindings(env *config.TestEnvironment) { rbTester := rolebinding.NewRoleBinding(common.DefaultTimeout, serviceAccountName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), rbTester, []reel.Handler{rbTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - if rbTester.Result() == tnf.FAILURE { - log.Info("RoleBindings: ", rbTester.GetRoleBindings()) - } - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidateWithFailureCallback(func() { log.Info("RoleBindings: ", rbTester.GetRoleBindings()) }) } }) } +//nolint:dupl func testClusterRoleBindings(env *config.TestEnvironment) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRoleBindingsBestPracticesIdentifier) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodClusterRoleBindingsBestPracticesIdentifier) ginkgo.It(testID, func() { ginkgo.By("Should not have ClusterRoleBindings") for _, podUnderTest := range env.PodsUnderTest { @@ -221,12 +212,7 @@ func testClusterRoleBindings(env *config.TestEnvironment) { crbTester := clusterrolebinding.NewClusterRoleBinding(common.DefaultTimeout, serviceAccountName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), crbTester, []reel.Handler{crbTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - if crbTester.Result() == tnf.FAILURE { - log.Info("ClusterRoleBindings: ", crbTester.GetClusterRoleBindings()) - } - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + test.RunAndValidateWithFailureCallback(func() { log.Info("ClusterRoleBindings: ", crbTester.GetClusterRoleBindings()) }) } }) } diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index 456583910..ee064da6e 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -26,7 +26,6 @@ import ( "github.com/onsi/gomega" log "github.com/sirupsen/logrus" - "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" ) @@ -53,13 +52,6 @@ func GetContext() *interactive.Context { return context } -// RunAndValidateTest runs the test and checks the result -func RunAndValidateTest(test *tnf.Test) { - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) -} - // IsMinikube returns true when the env var is set, OCP only test would be skipped based on this flag func IsMinikube() bool { b, _ := strconv.ParseBool(os.Getenv("TNF_MINIKUBE_ONLY")) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 5f12dcf84..4f33b49af 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -81,22 +81,20 @@ var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { defer results.RecordResult(identifiers.TestExtractNodeInformationIdentifier) context := common.GetContext() - test, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) + tester, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(jsonParseResult).ToNot(gomega.BeNil()) gomega.Expect(jsonParseResult.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - result, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(test).ToNot(gomega.BeNil()) - genericTest := (*test).(*generic.Generic) + test.RunAndValidate() + + genericTest := (*tester).(*generic.Generic) gomega.Expect(genericTest).ToNot(gomega.BeNil()) matches := genericTest.Matches gomega.Expect(len(matches)).To(gomega.Equal(1)) @@ -197,9 +195,7 @@ func listNodeCniPlugins(nodeName string) []CniPlugin { tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() gomega.Expect(len(tester.Processed)%2 == 0).To(gomega.BeTrue()) for i := 0; i < len(tester.Processed); i += 2 { result = append(result, CniPlugin{ @@ -215,9 +211,7 @@ func testOcpVersion() { tester := clusterversion.NewClusterVersion(defaultTestTimeout) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + test.RunAndValidate() versionsOcp = tester.GetVersions() } @@ -263,9 +257,7 @@ func getNodeLscpu(nodeName string) map[string]string { tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() for _, line := range tester.Processed { fields := strings.SplitN(line, ":", numSplitSubstrings) result[fields[0]] = strings.TrimSpace(fields[1]) @@ -281,9 +273,7 @@ func getNodeIfconfig(nodeName string) map[string][]string { tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() deviceName := "" for _, line := range tester.Processed { if line == "" { @@ -305,9 +295,7 @@ func getNodeLsblk(nodeName string) interface{} { tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, false, false) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() result := map[string]interface{}{} err = json.Unmarshal([]byte(tester.Raw), &result) gomega.Expect(err).To(gomega.BeNil()) @@ -320,9 +308,7 @@ func getNodeLspci(nodeName string) []string { tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() return tester.Processed } @@ -332,20 +318,18 @@ func listClusterCSIInfo() { ginkgo.Skip("CSI is not checked in minikube") } context := common.GetContext() - test, handlers, result, err := generic.NewGenericFromJSONFile(relativeCsiDriverTestPath, common.RelativeSchemaPath) + tester, handlers, result, err := generic.NewGenericFromJSONFile(relativeCsiDriverTestPath, common.RelativeSchemaPath) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(test).ToNot(gomega.BeNil()) - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - genericTest := (*test).(*generic.Generic) + gomega.Expect(test).ToNot(gomega.BeNil()) + test.RunAndValidate() + genericTest := (*tester).(*generic.Generic) gomega.Expect(genericTest).ToNot(gomega.BeNil()) matches := genericTest.Matches gomega.Expect(len(matches)).To(gomega.Equal(1)) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 90109873f..b28a48ce1 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -167,8 +167,7 @@ func runScalingTest(deployment configsections.Deployment) { handler := scaling.NewScaling(common.DefaultTimeout, deployment.Namespace, deployment.Name, deployment.Replicas) test, err := tnf.NewTest(common.GetContext().GetExpecter(), handler, []reel.Handler{handler}, common.GetContext().GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) - + test.RunAndValidate() // Wait until the deployment is ready waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } @@ -220,13 +219,11 @@ func testNodeSelector(env *config.TestEnvironment) { tester := nodeselector.NewNodeSelector(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - if testResult != tnf.SUCCESS { + test.RunAndValidateWithFailureCallback(func() { msg := fmt.Sprintf("The pod specifies nodeSelector/nodeAffinity field, you might want to change it, %s %s", podNamespace, podName) log.Warn(msg) infoWriter(msg) - } + }) } }) } @@ -245,9 +242,7 @@ func testGracePeriod(env *config.TestEnvironment) { tester := graceperiod.NewGracePeriod(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() gracePeriod := tester.GetGracePeriod() if gracePeriod == defaultTerminationGracePeriod { msg := fmt.Sprintf("%s %s has terminationGracePeriod set to %d, you might want to change it", podNamespace, podName, defaultTerminationGracePeriod) @@ -278,21 +273,18 @@ func shutdownTest(podNamespace, podName string) { values["POD_NAMESPACE"] = podNamespace values["POD_NAME"] = podName values["GO_TEMPLATE_PATH"] = relativeShutdownTestDirectoryPath - test, handlers, result, err := generic.NewGenericFromMap(relativeShutdownTestPath, common.RelativeSchemaPath, values) + tester, handlers, result, err := generic.NewGenericFromMap(relativeShutdownTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(test).ToNot(gomega.BeNil()) + + test.RunAndValidate() } func testPodsRecreation(env *config.TestEnvironment) { @@ -355,7 +347,7 @@ func getDeploymentsNodes(namespace string) []node { tester := dn.NewDeploymentsNodes(common.DefaultTimeout, namespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() nodes := tester.GetNodes() gomega.Expect(nodes).NotTo(gomega.BeEmpty()) return sortNodesMap(nodes) @@ -367,7 +359,7 @@ func getDeployments(namespace string) (deployments dp.DeploymentMap, notReadyDep tester := dp.NewDeployments(common.DefaultTimeout, namespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() deployments = tester.GetDeployments() @@ -385,28 +377,25 @@ func drainNode(node string) { tester := dd.NewDeploymentsDrain(drainTimeout, node) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() } func uncordonNode(node string) { context := common.GetContext() values := make(map[string]interface{}) values["NODE"] = node - test, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, common.RelativeSchemaPath, values) + tester, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(test).ToNot(gomega.BeNil()) - - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(test).ToNot(gomega.BeNil()) + test.RunAndValidate() } // Pod antiaffinity test for all deployments @@ -436,19 +425,18 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { values["DEPLOYMENT_NAME"] = deployment values["DEPLOYMENT_NAMESPACE"] = podNamespace infoWriter := tnf.CreateTestExtraInfoWriter() - test, handlers, result, err := generic.NewGenericFromMap(relativePodTestPath, common.RelativeSchemaPath, values) + tester, handlers, result, err := generic.NewGenericFromMap(relativePodTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(test).ToNot(gomega.BeNil()) - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := tester.Run() - if testResult != tnf.SUCCESS { + test.RunAndValidateWithFailureCallback(func() { if replica > 1 { msg := fmt.Sprintf("The deployment replica count is %d, but a podAntiAffinity rule is not defined, "+ "you might want to change it in deployment %s in namespace %s", replica, deployment, podNamespace) @@ -461,9 +449,7 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { log.Warn(msg) infoWriter(msg) } - } - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + }) } func testOwner(env *config.TestEnvironment) { @@ -479,9 +465,7 @@ func testOwner(env *config.TestEnvironment) { tester := owners.NewOwners(common.DefaultTimeout, podNamespace, podName) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() } }) } diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 88b029f43..e61ed98e4 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -126,7 +126,7 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count pingTester := ping.NewPing(common.DefaultTimeout, targetPodIPAddress, count) test, err := tnf.NewTest(initiatingPodOc.GetExpecter(), pingTester, []reel.Handler{pingTester}, initiatingPodOc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() transmitted, received, errors := pingTester.GetStats() gomega.Expect(received).To(gomega.Equal(transmitted)) gomega.Expect(errors).To(gomega.BeZero()) @@ -143,9 +143,7 @@ func testNodePort(env *config.TestEnvironment) { tester := nodeport.NewNodePort(common.DefaultTimeout, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() } }) } diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index c10f618e1..ec14ec2df 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -73,18 +73,16 @@ func loggingTest(c configsections.ContainerIdentifier) { values["POD_NAMESPACE"] = c.Namespace values["POD_NAME"] = c.PodName values["CONTAINER_NAME"] = c.ContainerName - test, handlers, result, err := generic.NewGenericFromMap(relativeLoggingTestPath, common.RelativeSchemaPath, values) + tester, handlers, result, err := generic.NewGenericFromMap(relativeLoggingTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(test).ToNot(gomega.BeNil()) + + test.RunAndValidate() } diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index ced61b800..732e706f5 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -107,21 +107,19 @@ func testOperatorIsInstalledViaOLM(subscriptionName, subscriptionNamespace strin values := make(map[string]interface{}) values["SUBSCRIPTION_NAME"] = subscriptionName values["SUBSCRIPTION_NAMESPACE"] = subscriptionNamespace - test, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, relativeSchemaPath, values) + tester, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, relativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(test).ToNot(gomega.BeNil()) - - tester, err := tnf.NewTest(context.GetExpecter(), *test, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(test).ToNot(gomega.BeNil()) + + test.RunAndValidate() } func itRunsTestsOnOperator(env *config.TestEnvironment) { @@ -161,9 +159,7 @@ func runTestsOnOperator(env *config.TestEnvironment, testCase testcases.BaseTest test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + test.RunAndValidate() } }) } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index a76f24b1f..da3264fd9 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -94,9 +94,7 @@ func testContainerIsRedHatRelease(cut *config.Container) { versionTester := redhat.NewRelease(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), versionTester, []reel.Handler{versionTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() } // testContainersFsDiff test that all CUT didn't install new packages are starting @@ -111,40 +109,38 @@ func testContainersFsDiff(env *config.TestEnvironment) { context := cut.Oc nodeName := cut.ContainerConfiguration.NodeName ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) - testResult, err := testContainerFsDiff(nodeName, context) - if testResult != tnf.SUCCESS || err != nil { + test := newContainerFsDiffTest(nodeName, context) + test.RunWithFailureCallback(func() { badContainers = append(badContainers, containerName) ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) - } + }) } gomega.Expect(badContainers).To(gomega.BeNil()) }) }) } -// testContainerFsDiff test that the CUT didn't install new packages after starting, and report through Ginkgo. -func testContainerFsDiff(nodeName string, targetContainerOC *interactive.Oc) (int, error) { +// newContainerFsDiffTest test that the CUT didn't install new packages after starting, and report through Ginkgo. +func newContainerFsDiffTest(nodeName string, targetContainerOC *interactive.Oc) *tnf.Test { defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) targetContainerOC.GetExpecter() containerIDTester := containerid.NewContainerID(common.DefaultTimeout) test, err := tnf.NewTest(targetContainerOC.GetExpecter(), containerIDTester, []reel.Handler{containerIDTester}, targetContainerOC.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() containerID := containerIDTester.GetID() context := common.GetContext() fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID, nodeName) test, err = tnf.NewTest(context.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - return test.Run() + return test } func getMcKernelArguments(context *interactive.Context, mcName string) map[string]string { mcKernelArgumentsTester := mckernelarguments.NewMcKernelArguments(common.DefaultTimeout, mcName) test, err := tnf.NewTest(context.GetExpecter(), mcKernelArgumentsTester, []reel.Handler{mcKernelArgumentsTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() mcKernelArguments := mcKernelArgumentsTester.GetKernelArguments() var mcKernelArgumentsJSON []string err = json.Unmarshal([]byte(mcKernelArguments), &mcKernelArgumentsJSON) @@ -157,7 +153,7 @@ func getMcName(context *interactive.Context, nodeName string) string { mcNameTester := nodemcname.NewNodeMcName(common.DefaultTimeout, nodeName) test, err := tnf.NewTest(context.GetExpecter(), mcNameTester, []reel.Handler{mcNameTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() return mcNameTester.GetMcName() } @@ -165,7 +161,7 @@ func getPodNodeName(context *interactive.Context, podName, podNamespace string) podNameTester := podnodename.NewPodNodeName(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), podNameTester, []reel.Handler{podNameTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() return podNameTester.GetNodeName() } @@ -173,7 +169,7 @@ func getCurrentKernelCmdlineArgs(targetContainerOc *interactive.Oc) map[string]s currentKernelCmdlineArgsTester := currentkernelcmdlineargs.NewCurrentKernelCmdlineArgs(common.DefaultTimeout) test, err := tnf.NewTest(targetContainerOc.GetExpecter(), currentKernelCmdlineArgsTester, []reel.Handler{currentKernelCmdlineArgsTester}, targetContainerOc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() currnetKernelCmdlineArgs := currentKernelCmdlineArgsTester.GetKernelArguments() currentSplitKernelCmdlineArgs := strings.Split(currnetKernelCmdlineArgs, " ") return utils.ArgListToMap(currentSplitKernelCmdlineArgs) @@ -183,7 +179,7 @@ func getGrubKernelArgs(context *interactive.Context, nodeName string) map[string readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout, nodeName) test, err := tnf.NewTest(context.GetExpecter(), readBootConfigTester, []reel.Handler{readBootConfigTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() bootConfig := readBootConfigTester.GetBootConfig() splitBootConfig := strings.Split(bootConfig, "\n") @@ -226,7 +222,7 @@ func getSysctlConfigArgs(context *interactive.Context, nodeName string) map[stri sysctlAllConfigsArgsTester := sysctlallconfigsargs.NewSysctlAllConfigsArgs(common.DefaultTimeout, nodeName) test, err := tnf.NewTest(context.GetExpecter(), sysctlAllConfigsArgsTester, []reel.Handler{sysctlAllConfigsArgsTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - common.RunAndValidateTest(test) + test.RunAndValidate() sysctlAllConfigsArgs := sysctlAllConfigsArgsTester.GetSysctlAllConfigsArgs() return parseSysctlSystemOutput(sysctlAllConfigsArgs) @@ -297,12 +293,9 @@ func testTainted(env *config.TestEnvironment) { tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node.Name) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).NotTo(gomega.Equal(tnf.ERROR)) - gomega.Expect(err).To(gomega.BeNil()) - if testResult == tnf.FAILURE { + test.RunWithFailureCallback(func() { taintedNodes = append(taintedNodes, node.Name) - } + }) } gomega.Expect(taintedNodes).To(gomega.BeNil()) }) @@ -324,6 +317,7 @@ func testHugepages(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) ginkgo.It(testID, func() { defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) + var badNodes []string context := common.GetContext() for _, node := range env.Nodes { @@ -338,12 +332,10 @@ func testHugepages(env *config.TestEnvironment) { tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node.Name, nodeHugePagesSize, nodeHugePagesCount) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(err).To(gomega.BeNil()) - if testResult != tnf.SUCCESS { + test.RunWithFailureCallback(func() { badNodes = append(badNodes, node.Name) ginkgo.By(fmt.Sprintf("Node=%s hugepage config does not match machineconfig", node.Name)) - } + }) } gomega.Expect(badNodes).To(gomega.BeNil()) }) From e579c566b0969a8b32985137db3f7c0ac2af26f6 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Mon, 11 Oct 2021 07:59:44 -0500 Subject: [PATCH 087/344] Switch to go install (#389) --- .github/workflows/merge.yaml | 2 +- .github/workflows/pre-main.yaml | 8 ++++---- Makefile | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml index f7c507dcf..ab702b526 100644 --- a/.github/workflows/merge.yaml +++ b/.github/workflows/merge.yaml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v2 - name: Rebuild mocks - run: go get github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen && make mocks - name: Run Tests run: make test diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 87de35b7b..34a26fbea 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -40,7 +40,7 @@ jobs: ref: ${{ github.sha }} - name: Rebuild mocks - run: go get github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen && make mocks # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However # there does not appear to be a way to honor our existing .golangci.yml. For now, mimic developer behavior. @@ -66,7 +66,7 @@ jobs: ref: ${{ github.sha }} - name: Rebuild mocks - run: go get github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen && make mocks - name: Run Tests run: make test @@ -90,10 +90,10 @@ jobs: ref: ${{ github.sha }} - name: Execute `make mocks` - run: go get github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen && make mocks - name: Install ginkgo - run: go get -u github.com/onsi/ginkgo/ginkgo + run: go install github.com/onsi/ginkgo/ginkgo - name: Execute `make build` run: make build diff --git a/Makefile b/Makefile index 979e090b1..4eff7bca7 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,6 @@ update-deps: # Install build tools and other required software. install-tools: - go get github.com/onsi/ginkgo/ginkgo - go get github.com/onsi/gomega/... - go get github.com/golang/mock/mockgen + go install github.com/onsi/ginkgo/ginkgo + go install github.com/onsi/gomega/... + go install github.com/golang/mock/mockgen From 6b2d5c84412ac7bdff368b27666d158d2caf8f51 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 12 Oct 2021 08:28:37 -0500 Subject: [PATCH 088/344] Add unit tests wrapping exec commands (#388) --- Makefile | 8 +- pkg/config/autodiscover/csv_info.go | 7 +- pkg/config/autodiscover/csv_info_test.go | 3 +- pkg/config/autodiscover/deployment_info.go | 11 +- .../autodiscover/deployment_info_test.go | 81 +++++++- pkg/config/autodiscover/pod_info.go | 15 +- pkg/config/autodiscover/pod_info_test.go | 3 +- .../autodiscover/testdata/testdeploy.json | 184 ++++++++++++++++++ 8 files changed, 292 insertions(+), 20 deletions(-) create mode 100644 pkg/config/autodiscover/testdata/testdeploy.json diff --git a/Makefile b/Makefile index 4eff7bca7..710253c05 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ # Tasks provide shortcuts to common operations that occur frequently during # development. This includes running configured linters and executing unit tests +GO_PACKAGES=$(shell go list ./... | grep -v vendor) + .PHONY: build \ mocks \ clean \ @@ -33,7 +35,8 @@ run-container-tests \ clean-mocks \ update-deps \ - install-tools + install-tools \ + vet # Get default value of $GOBIN if not explicitly set ifeq (,$(shell go env GOBIN)) @@ -146,3 +149,6 @@ install-tools: go install github.com/onsi/ginkgo/ginkgo go install github.com/onsi/gomega/... go install github.com/golang/mock/mockgen + +vet: + go vet ${GO_PACKAGES} diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index e318df4ff..0fbf9ecbd 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -17,7 +17,6 @@ package autodiscover import ( - "encoding/json" "fmt" log "github.com/sirupsen/logrus" @@ -56,7 +55,7 @@ func (csv *CSVResource) GetAnnotationValue(annotationKey string, v interface{}) return fmt.Errorf("failed to find annotation '%s' on CSV '%s/%s'", annotationKey, csv.Metadata.Namespace, csv.Metadata.Name) } val := csv.Metadata.Annotations[annotationKey] - err = json.Unmarshal([]byte(val), v) + err = jsonUnmarshal([]byte(val), v) if err != nil { return csv.annotationUnmarshalError(annotationKey, err) } @@ -73,7 +72,7 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) - out, err := cmd.Output() + out, err := execCommandOutput(cmd) if err != nil { return nil, err } @@ -81,7 +80,7 @@ func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { log.Debug("Command: ", cmd) log.Debug(string(out)) var csvList CSVList - err = json.Unmarshal(out, &csvList) + err = jsonUnmarshal(out, &csvList) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/csv_info_test.go b/pkg/config/autodiscover/csv_info_test.go index f7d05fd7e..51e48a119 100644 --- a/pkg/config/autodiscover/csv_info_test.go +++ b/pkg/config/autodiscover/csv_info_test.go @@ -17,7 +17,6 @@ package autodiscover import ( - "encoding/json" "log" "os" "path" @@ -39,7 +38,7 @@ func loadCSVResource(filePath string) (csv CSVResource) { if err != nil { log.Fatalf("error (%s) loading CSVResource %s for testing", err, filePath) } - err = json.Unmarshal(contents, &csv) + err = jsonUnmarshal(contents, &csv) if err != nil { log.Fatalf("error (%s) loading CSVResource %s for testing", err, filePath) } diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index 2e4f8a834..7728beed1 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -28,6 +28,13 @@ const ( resourceTypeDeployment = "deployment" ) +var ( + jsonUnmarshal = json.Unmarshal + execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { + return cmd.Output() + } +) + // DeploymentList holds the data from an `oc get deployments -o json` command type DeploymentList struct { Items []DeploymentResource `json:"items"` @@ -75,13 +82,13 @@ func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsection cmd := exec.Command("bash", "-c", ocCmd) - out, err := cmd.Output() + out, err := execCommandOutput(cmd) if err != nil { return nil, err } var deploymentList DeploymentList - err = json.Unmarshal(out, &deploymentList.Items) + err = jsonUnmarshal(out, &deploymentList.Items) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/deployment_info_test.go b/pkg/config/autodiscover/deployment_info_test.go index 192fba808..c94f18d72 100644 --- a/pkg/config/autodiscover/deployment_info_test.go +++ b/pkg/config/autodiscover/deployment_info_test.go @@ -18,20 +18,25 @@ package autodiscover import ( "encoding/json" + "errors" "log" "os" + "os/exec" "path" "testing" "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/config/configsections" ) const ( testDeploymentFile = "testdeployment.json" + testJQFile = "testdeploy.json" ) var ( testDeploymentFilePath = path.Join(filePath, testDeploymentFile) + testJQFilePath = path.Join(filePath, testJQFile) ) func loadDeployment(filePath string) (deployment DeploymentResource) { @@ -39,7 +44,7 @@ func loadDeployment(filePath string) (deployment DeploymentResource) { if err != nil { log.Fatalf("error (%s) loading DeploymentResource %s for testing", err, filePath) } - err = json.Unmarshal(contents, &deployment) + err = jsonUnmarshal(contents, &deployment) if err != nil { log.Fatalf("error (%s) unmarshalling DeploymentResource %s for testing", err, filePath) } @@ -57,3 +62,77 @@ func TestPodGetAnnotationValue1(t *testing.T) { assert.Equal(t, 1, len(labels)) assert.Equal(t, "test", labels["app"]) } + +//nolint:funlen +func TestGetTargetDeploymentByNamespace(t *testing.T) { + testCases := []struct { + badExec bool + execErr error + badJSONUnmarshal bool + jsonErr error + }{ + { // no failures + badExec: false, + badJSONUnmarshal: false, + }, + { // failure to exec + badExec: true, + execErr: errors.New("this is an error"), + badJSONUnmarshal: false, + jsonErr: nil, + }, + { // failure to jsonUnmarshal + badExec: false, + execErr: nil, + badJSONUnmarshal: true, + jsonErr: errors.New("this is an error"), + }, + } + + origExecFunc := execCommandOutput + + for _, tc := range testCases { + // Setup the mock functions + if tc.badExec { + execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { + return nil, tc.execErr + } + } else { + execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { + contents, err := os.ReadFile(testJQFilePath) + assert.Nil(t, err) + return contents, nil + } + } + if tc.badJSONUnmarshal { + jsonUnmarshal = func(data []byte, v interface{}) error { + return tc.jsonErr + } + } else { + // use the "real" function + jsonUnmarshal = json.Unmarshal + } + + // Run the function and compare the list output + list, err := GetTargetDeploymentsByNamespace("test", configsections.Label{ + Prefix: "prefix1", + Name: "name1", + Value: "value1", + }) + if !tc.badExec && !tc.badJSONUnmarshal { + assert.NotNil(t, list) + assert.Equal(t, "my-test1", list.Items[0].Metadata.Name) + } + + // Assert the errors and cleanup + if tc.badExec { + assert.NotNil(t, err) + execCommandOutput = origExecFunc + } + + if tc.badJSONUnmarshal { + assert.NotNil(t, err) + jsonUnmarshal = json.Unmarshal + } + } +} diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index ae4f6cbc7..9563b192b 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -17,7 +17,6 @@ package autodiscover import ( - "encoding/json" "fmt" log "github.com/sirupsen/logrus" @@ -82,7 +81,7 @@ func (pr *PodResource) GetAnnotationValue(annotationKey string, v interface{}) ( return fmt.Errorf("failed to find annotation '%s' on pod '%s/%s'", annotationKey, pr.Metadata.Namespace, pr.Metadata.Name) } val := pr.Metadata.Annotations[annotationKey] - err = json.Unmarshal([]byte(val), v) + err = jsonUnmarshal([]byte(val), v) if err != nil { return pr.annotationUnmarshalError(annotationKey, err) } @@ -97,12 +96,12 @@ func (pr *PodResource) GetAnnotationValue(annotationKey string, v interface{}) ( func (pr *PodResource) getDefaultNetworkDeviceFromAnnotations() (iface string, err error) { // Note: The `GetAnnotationValue` method does not distinguish between bad encoding and a missing annotation, which is needed here. if val, present := pr.Metadata.Annotations[namespacedDefaultNetworkInterfaceKey]; present { - err = json.Unmarshal([]byte(val), &iface) + err = jsonUnmarshal([]byte(val), &iface) return } if val, present := pr.Metadata.Annotations[cniNetworksStatusKey]; present { var cniInfo []cniNetworkInterface - err = json.Unmarshal([]byte(val), &cniInfo) + err = jsonUnmarshal([]byte(val), &cniInfo) if err != nil { return "", pr.annotationUnmarshalError(cniNetworksStatusKey, err) } @@ -124,12 +123,12 @@ func (pr *PodResource) getDefaultNetworkDeviceFromAnnotations() (iface string, e func (pr *PodResource) getPodIPs() (ips []string, err error) { // Note: The `GetAnnotationValue` method does not distinguish between bad encoding and a missing annotation, which is needed here. if val, present := pr.Metadata.Annotations[namespacedIPsKey]; present { - err = json.Unmarshal([]byte(val), &ips) + err = jsonUnmarshal([]byte(val), &ips) return } if val, present := pr.Metadata.Annotations[cniNetworksStatusKey]; present { var cniInfo []cniNetworkInterface - err = json.Unmarshal([]byte(val), &cniInfo) + err = jsonUnmarshal([]byte(val), &cniInfo) if err != nil { return nil, pr.annotationUnmarshalError(cniNetworksStatusKey, err) } @@ -155,7 +154,7 @@ func (pr *PodResource) annotationUnmarshalError(annotationKey string, err error) func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, error) { cmd := makeGetCommand(resourceTypePods, buildLabelQuery(label), namespace) - out, err := cmd.Output() + out, err := execCommandOutput(cmd) if err != nil { return nil, err } @@ -163,7 +162,7 @@ func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, err log.Debug("Command: ", cmd) log.Debug(string(out)) var podList PodList - err = json.Unmarshal(out, &podList) + err = jsonUnmarshal(out, &podList) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/pod_info_test.go b/pkg/config/autodiscover/pod_info_test.go index 6e34eb753..8e3e30a1e 100644 --- a/pkg/config/autodiscover/pod_info_test.go +++ b/pkg/config/autodiscover/pod_info_test.go @@ -17,7 +17,6 @@ package autodiscover import ( - "encoding/json" "log" "os" "path" @@ -42,7 +41,7 @@ func loadPodResource(filePath string) (pod PodResource) { if err != nil { log.Fatalf("error (%s) loading PodResource %s for testing", err, filePath) } - err = json.Unmarshal(contents, &pod) + err = jsonUnmarshal(contents, &pod) if err != nil { log.Fatalf("error (%s) loading PodResource %s for testing", err, filePath) } diff --git a/pkg/config/autodiscover/testdata/testdeploy.json b/pkg/config/autodiscover/testdata/testdeploy.json new file mode 100644 index 000000000..afd58d3f4 --- /dev/null +++ b/pkg/config/autodiscover/testdata/testdeploy.json @@ -0,0 +1,184 @@ +[{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "deployment.kubernetes.io/revision": "1" + }, + "creationTimestamp": "2021-10-11T15:22:41Z", + "generation": 1, + "labels": { + "app": "my-test1" + }, + "managedFields": [{ + "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:app": {} + } + }, + "f:spec": { + "f:progressDeadlineSeconds": {}, + "f:replicas": {}, + "f:revisionHistoryLimit": {}, + "f:selector": {}, + "f:strategy": { + "f:rollingUpdate": { + ".": {}, + "f:maxSurge": {}, + "f:maxUnavailable": {} + }, + "f:type": {} + }, + "f:template": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:app": {}, + "f:prefix1/name1": {} + } + }, + "f:spec": { + "f:containers": { + "k:{\"name\":\"nginx\"}": { + ".": {}, + "f:image": {}, + "f:imagePullPolicy": {}, + "f:name": {}, + "f:resources": {}, + "f:terminationMessagePath": {}, + "f:terminationMessagePolicy": {} + } + }, + "f:dnsPolicy": {}, + "f:restartPolicy": {}, + "f:schedulerName": {}, + "f:securityContext": {}, + "f:terminationGracePeriodSeconds": {} + } + } + } + }, + "manager": "oc", + "operation": "Update", + "time": "2021-10-11T15:22:41Z" + }, + { + "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:deployment.kubernetes.io/revision": {} + } + }, + "f:status": { + "f:availableReplicas": {}, + "f:conditions": { + ".": {}, + "k:{\"type\":\"Available\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + }, + "k:{\"type\":\"Progressing\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + } + }, + "f:observedGeneration": {}, + "f:readyReplicas": {}, + "f:replicas": {}, + "f:updatedReplicas": {} + } + }, + "manager": "kube-controller-manager", + "operation": "Update", + "subresource": "status", + "time": "2021-10-11T15:22:44Z" + } + ], + "name": "my-test1", + "namespace": "test", + "resourceVersion": "15460", + "uid": "bc6b1ac5-ee45-47a1-bc41-1258e0cf30a3" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "app": "my-test1" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": "25%", + "maxUnavailable": "25%" + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "app": "my-test1", + "prefix1/name1": "value1" + } + }, + "spec": { + "containers": [{ + "image": "nginx", + "imagePullPolicy": "Always", + "name": "nginx", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File" + }], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "terminationGracePeriodSeconds": 30 + } + } + }, + "status": { + "availableReplicas": 1, + "conditions": [{ + "lastTransitionTime": "2021-10-11T15:22:44Z", + "lastUpdateTime": "2021-10-11T15:22:44Z", + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2021-10-11T15:22:41Z", + "lastUpdateTime": "2021-10-11T15:22:44Z", + "message": "ReplicaSet \"my-test1-57dbcf59f6\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 1, + "readyReplicas": 1, + "replicas": 1, + "updatedReplicas": 1 + } +}] From e3c35ca3f158c0845b9e4c7d6e60a212aaecb94f Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 12 Oct 2021 10:45:43 -0500 Subject: [PATCH 089/344] Add yamllint action (#390) --- .github/workflows/pre-main.yaml | 19 +++++++++++++ examples/generic/template/ping.values.yaml | 2 +- examples/pty/ssh.json.tpl.values.yaml | 2 +- pkg/config/testdata/tnf_test_config.yml | 8 +++--- .../handlers/generic/testdata/bad_yaml.yaml | 2 +- .../testdata/ssh.json.tpl.values.bad.nonyaml | 2 +- .../testdata/ssh.json.tpl.values.bad.yaml.tpl | 2 +- .../testcases/files/cnf/gatherpodfacts.yml | 28 +++++++++---------- pkg/tnf/testcases/files/cnf/privilegedpod.yml | 2 +- .../testcases/files/cnf/privilegedroles.yml | 2 +- .../files/operator/operatorstatus.yml | 1 - test-network-function/testconfigure.yml | 2 +- test-network-function/tnf_config.yml | 2 +- 13 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 34a26fbea..b641807e5 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -50,6 +50,25 @@ jobs: - name: make lint run: make lint + yamllint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v3 + with: + config_data: | + extends: default + rules: + line-length: + level: warning + trailing-spaces: + level: warning + brackets: + level: warning + empty-lines: + level: warning + unit-tests: name: Run Unit Tests runs-on: ubuntu-20.04 diff --git a/examples/generic/template/ping.values.yaml b/examples/generic/template/ping.values.yaml index e68066ff0..40e52852c 100644 --- a/examples/generic/template/ping.values.yaml +++ b/examples/generic/template/ping.values.yaml @@ -1 +1 @@ -HOST: 192.168.1.1 \ No newline at end of file +HOST: 192.168.1.1 diff --git a/examples/pty/ssh.json.tpl.values.yaml b/examples/pty/ssh.json.tpl.values.yaml index 7d746ea1a..4540132a7 100644 --- a/examples/pty/ssh.json.tpl.values.yaml +++ b/examples/pty/ssh.json.tpl.values.yaml @@ -1,4 +1,4 @@ SSH_ARGS: - 192.168.1.5 - -l - - pi \ No newline at end of file + - pi diff --git a/pkg/config/testdata/tnf_test_config.yml b/pkg/config/testdata/tnf_test_config.yml index 69f3e2e22..75ab788c1 100644 --- a/pkg/config/testdata/tnf_test_config.yml +++ b/pkg/config/testdata/tnf_test_config.yml @@ -38,12 +38,12 @@ testPartner: multusIpAddresses: - 10.217.0.29 testOrchestrator: - namespace: default - podName: partner - containerName: partner + namespace: default + podName: partner + containerName: partner certifiedcontainerinfo: - name: nginx-116 # working example repository: rhel8 certifiedoperatorinfo: - name: etcd-operator - organization: redhat-marketplace \ No newline at end of file + organization: redhat-marketplace diff --git a/pkg/tnf/handlers/generic/testdata/bad_yaml.yaml b/pkg/tnf/handlers/generic/testdata/bad_yaml.yaml index 45097746d..7225d2e1a 100644 --- a/pkg/tnf/handlers/generic/testdata/bad_yaml.yaml +++ b/pkg/tnf/handlers/generic/testdata/bad_yaml.yaml @@ -1 +1 @@ -Not a Key/Value Pair!!! \ No newline at end of file +Not a Key/Value Pair!!! diff --git a/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.nonyaml b/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.nonyaml index 08a3a39f2..e568ca35f 100644 --- a/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.nonyaml +++ b/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.nonyaml @@ -1,3 +1,3 @@ { "notyaml" -} \ No newline at end of file +} diff --git a/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.yaml.tpl b/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.yaml.tpl index d9e27d1a2..4b271d455 100644 --- a/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.yaml.tpl +++ b/pkg/tnf/interactive/testdata/ssh.json.tpl.values.bad.yaml.tpl @@ -1,2 +1,2 @@ bad: - - keys \ No newline at end of file + - keys diff --git a/pkg/tnf/testcases/files/cnf/gatherpodfacts.yml b/pkg/tnf/testcases/files/cnf/gatherpodfacts.yml index cc8b7552b..4632b2a41 100644 --- a/pkg/tnf/testcases/files/cnf/gatherpodfacts.yml +++ b/pkg/tnf/testcases/files/cnf/gatherpodfacts.yml @@ -1,12 +1,12 @@ testcase: - name: "NAME" - skiptest: false - command: "oc get pod %s -n %s -o json | jq -r '.metadata.name'" - action: "allow" - resulttype: "string" - expectedType: "regex" - expectedstatus: - - "ALLOW_ALL" + skiptest: false + command: "oc get pod %s -n %s -o json | jq -r '.metadata.name'" + action: "allow" + resulttype: "string" + expectedType: "regex" + expectedstatus: + - "ALLOW_ALL" - name: "CONTAINER_COUNT" skiptest: false command: "oc get pod %s -n %s -o json | jq -r '.spec.containers | length'" @@ -16,10 +16,10 @@ testcase: expectedstatus: - "DIGIT" - name: "SERVICE_ACCOUNT_NAME" - skiptest: false - command: "oc get pod %s -n %s -o json | jq -r '.spec.serviceAccountName'" - action: "allow" - resulttype: "string" - expectedType: "regex" - expectedstatus: - - "ALLOW_ALL" \ No newline at end of file + skiptest: false + command: "oc get pod %s -n %s -o json | jq -r '.spec.serviceAccountName'" + action: "allow" + resulttype: "string" + expectedType: "regex" + expectedstatus: + - "ALLOW_ALL" diff --git a/pkg/tnf/testcases/files/cnf/privilegedpod.yml b/pkg/tnf/testcases/files/cnf/privilegedpod.yml index 8019bc2e5..91cac1843 100644 --- a/pkg/tnf/testcases/files/cnf/privilegedpod.yml +++ b/pkg/tnf/testcases/files/cnf/privilegedpod.yml @@ -65,4 +65,4 @@ testcase: action: allow expectedType: "regex" expectedstatus: - - NULL_FALSE \ No newline at end of file + - NULL_FALSE diff --git a/pkg/tnf/testcases/files/cnf/privilegedroles.yml b/pkg/tnf/testcases/files/cnf/privilegedroles.yml index ffb99ac30..f59870647 100644 --- a/pkg/tnf/testcases/files/cnf/privilegedroles.yml +++ b/pkg/tnf/testcases/files/cnf/privilegedroles.yml @@ -15,4 +15,4 @@ testcase: expectedType: "function" expectedstatus: - "FN_SERVICE_ACCOUNT_NAME" - - "null" \ No newline at end of file + - "null" diff --git a/pkg/tnf/testcases/files/operator/operatorstatus.yml b/pkg/tnf/testcases/files/operator/operatorstatus.yml index 2be8fa47c..2e7a958ec 100644 --- a/pkg/tnf/testcases/files/operator/operatorstatus.yml +++ b/pkg/tnf/testcases/files/operator/operatorstatus.yml @@ -15,4 +15,3 @@ testcase: expectedtype: "string" expectedstatus: - "EMPTY" - diff --git a/test-network-function/testconfigure.yml b/test-network-function/testconfigure.yml index 6cd2c290b..c9615e31e 100644 --- a/test-network-function/testconfigure.yml +++ b/test-network-function/testconfigure.yml @@ -16,4 +16,4 @@ operatortest: - name: "OPERATOR_STATUS" tests: - "CSV_INSTALLED" - - "CSV_SCC" \ No newline at end of file + - "CSV_SCC" diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 725a38cde..7710d1d9c 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -58,4 +58,4 @@ certifiedcontainerinfo: repository: rhel8 certifiedoperatorinfo: - name: etcd - organization: community-operators # working example \ No newline at end of file + organization: community-operators # working example From 94f1753842592dfe2450b10b237c9323f39282cd Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 12 Oct 2021 13:27:30 -0400 Subject: [PATCH 090/344] Include vet in workflow (#394) * Include vet in workflow * Update pre-main.yaml --- .github/workflows/pre-main.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index b641807e5..557ad2f17 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -25,7 +25,7 @@ env: jobs: lint: - name: Run Linter + name: Run Linter and Vet runs-on: ubuntu-20.04 steps: @@ -49,6 +49,9 @@ jobs: - name: make lint run: make lint + + - name: make vet + run: make vet yamllint: runs-on: ubuntu-20.04 From 2c05996b6ba5088e70bb9f4c45b249a8573475c5 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 12 Oct 2021 13:10:11 -0500 Subject: [PATCH 091/344] Update ginkgo to 1.16.5 (#393) --- go.mod | 7 +++---- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 97b48a805..1643b179f 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,19 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/basgys/goxml2json v1.1.0 github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/golang/mock v1.6.0 github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f github.com/kr/pretty v0.2.1 // indirect - github.com/onsi/ginkgo v1.16.4 + github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.16.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 github.com/test-network-function/test-network-function-claim v1.0.4 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect google.golang.org/grpc v1.41.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 06ae2a522..e9cf999a2 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -80,8 +78,9 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -220,8 +219,9 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= @@ -321,7 +321,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -445,8 +444,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 0738b512c743f14af0538edff22ea40ae17fcd4f Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Tue, 12 Oct 2021 21:20:36 +0300 Subject: [PATCH 092/344] Replace exec.Command with Expecter (#353) * Replace exec.Command with Expecter * Replace exec.Command with Expecter. * Update Replace exe.command with expecter * Fix the error and change the timeout * Resolved comment of Gonzalo * Update resolved comment of Gonzalo * Resolved comment of Jun * resolved comment of Jun and Gonzalo * changed the name of attribute in function NewGetCommand(from ocCommand to command) * Update resolved comment of Gonzalo * Update resolved comment of Jun * for David * Fixed the error * Resolved comments of Gonzalo. * Resolved conflicts * Resolved comment of Brandon * Update Resolved comment of Brandon Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- pkg/config/autodiscover/autodiscover.go | 60 ++++++++++-- pkg/config/autodiscover/csv_info.go | 9 +- pkg/config/autodiscover/pod_info.go | 10 +- pkg/tnf/handlers/command/command.json | 21 +++++ pkg/tnf/handlers/command/command_test.go | 113 +++++++++++++++++++++++ pkg/tnf/handlers/command/doc.go | 17 ++++ pkg/tnf/identifier/identifiers.go | 17 ++++ 7 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 pkg/tnf/handlers/command/command.json create mode 100644 pkg/tnf/handlers/command/command_test.go create mode 100644 pkg/tnf/handlers/command/doc.go diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 74dab1fa7..64291374f 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -19,20 +19,37 @@ package autodiscover import ( "fmt" "os" - "os/exec" + "path" "strconv" + "time" + "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/test-network-function/common" ) const ( disableAutodiscoverEnvVar = "TNF_DISABLE_CONFIG_AUTODISCOVER" tnfLabelPrefix = "test-network-function.com" labelTemplate = "%s/%s" - // anyLabelValue is the value that will allow any value for a label when building the label query. - anyLabelValue = "" + anyLabelValue = "" + ocCommand = "oc get %s -n %s -o json -l %s" + ocCommandTimeOut = time.Second * 10 +) + +var ( + // TestFile is the file location of the command.json test case relative to the project root. + TestFile = path.Join("pkg", "tnf", "handlers", "command", "command.json") + + // pathToTestFile is the relative path to the command.json test case. + pathToTestFile = path.Join(common.PathRelativeToRoot, TestFile) + + // commandDriver stores the csi driver JSON output. + commandDriver = make(map[string]interface{}) ) // PerformAutoDiscovery checks the environment variable to see if autodiscovery should be performed @@ -60,12 +77,37 @@ func buildLabelQuery(label configsections.Label) string { return fullLabelName } -func makeGetCommand(resourceType, labelQuery, namespace string) *exec.Cmd { - // TODO: shell expecter - cmd := exec.Command("oc", "get", resourceType, "-n", namespace, "-o", "json", "-l", labelQuery) - log.Debug("Issuing get command ", cmd.Args) - - return cmd +func executeOcGetCommand(resourceType, labelQuery, namespace string) (string, error) { + ocCommandtoExecute := fmt.Sprintf(ocCommand, resourceType, namespace, labelQuery) + values := make(map[string]interface{}) + values["COMMAND"] = ocCommandtoExecute + values["TIMEOUT"] = ocCommandTimeOut.Nanoseconds() + context := common.GetContext() + test, handler, result, err := generic.NewGenericFromMap(pathToTestFile, common.RelativeSchemaPath, values) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handler).ToNot(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + + tester, err := tnf.NewTest(context.GetExpecter(), *test, handler, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + testResult, err := tester.Run() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + + genericTest := (*test).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + match := genericTest.GetMatches()[0] + err = jsonUnmarshal([]byte(match.Match), &commandDriver) + gomega.Expect(err).To(gomega.BeNil()) + return match.Match, err } // getContainersByLabel builds `config.Container`s from containers in pods matching a label. diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index 0fbf9ecbd..74fcdfa6d 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -70,17 +70,16 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error // GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that // label will be returned, regardless of the labels value. func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { - cmd := makeGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) + out, err := executeOcGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) - out, err := execCommandOutput(cmd) if err != nil { return nil, err } log.Debug("JSON output for all pods labeled with: ", labelName) - log.Debug("Command: ", cmd) - log.Debug(string(out)) + log.Debug("Command: ", out) + var csvList CSVList - err = jsonUnmarshal(out, &csvList) + err = jsonUnmarshal([]byte(out), &csvList) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 9563b192b..43355ef3d 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -152,17 +152,17 @@ func (pr *PodResource) annotationUnmarshalError(annotationKey string, err error) // GetPodsByLabel will return all pods with a given label value. If `labelValue` is an empty string, all pods with that // label will be returned, regardless of the labels value. func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, error) { - cmd := makeGetCommand(resourceTypePods, buildLabelQuery(label), namespace) + out, err := executeOcGetCommand(resourceTypePods, buildLabelQuery(label), namespace) - out, err := execCommandOutput(cmd) if err != nil { return nil, err } + log.Debug("JSON output for all pods labeled with: ", label) - log.Debug("Command: ", cmd) - log.Debug(string(out)) + log.Debug("Command: ", out) + var podList PodList - err = jsonUnmarshal(out, &podList) + err = jsonUnmarshal([]byte(out), &podList) if err != nil { return nil, err } diff --git a/pkg/tnf/handlers/command/command.json b/pkg/tnf/handlers/command/command.json new file mode 100644 index 000000000..6c74aafd0 --- /dev/null +++ b/pkg/tnf/handlers/command/command.json @@ -0,0 +1,21 @@ + +{ + "identifier" : { + "url" : "http://test-network-function.com/tests/command", + "version": "v1.0.0" + }, + "description": "Handler to execute user custom commands.", + "testResult": 0, + "testTimeout": {{ .TIMEOUT}}, + "reelFirstStep": { + "execute": "{{ .COMMAND}}", + "expect":[ "(?m).*"], + "timeout": {{ .TIMEOUT}} + }, + "resultContexts":[ + { + "pattern": "(?m).*", + "defaultResult": 1 + } + ] + } \ No newline at end of file diff --git a/pkg/tnf/handlers/command/command_test.go b/pkg/tnf/handlers/command/command_test.go new file mode 100644 index 000000000..a2dc6bfbf --- /dev/null +++ b/pkg/tnf/handlers/command/command_test.go @@ -0,0 +1,113 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package command + +import ( + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/xeipuuv/gojsonschema" +) + +const ( + testTimeoutDuration = time.Second * 1 +) + +var ( + genericTestSchemaFile = path.Join("schemas", "generic-test.schema.json") + checkSubFilename = "command.json" + expectedPassPattern = "(?m).*" + pathRelativeToRoot = path.Join("..", "..", "..", "..") + pathToTestSchemaFile = path.Join(pathRelativeToRoot, genericTestSchemaFile) + testCommand = "oc get pods -n tnf" +) + +func createTest() (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { + values := make(map[string]interface{}) + values["COMMAND"] = testCommand + values["TIMEOUT"] = testTimeoutDuration.Nanoseconds() + return generic.NewGenericFromMap(checkSubFilename, pathToTestSchemaFile, values) +} +func TestCommand_Args(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Nil(t, (*test).Args()) +} + +func TestCommand_GetIdentifier(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Equal(t, identifier.CommandIdentifier, (*test).GetIdentifier()) +} + +func TestCommand_ReelFirst(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + step := handler.ReelFirst() + assert.Equal(t, testCommand, step.Execute) + assert.Contains(t, step.Expect, expectedPassPattern) + assert.Equal(t, testTimeoutDuration, step.Timeout) +} + +func TestCommand_ReelEOF(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + // just ensure there isn't a panic + handler.ReelEOF() +} + +func TestCommand_ReelTimeout(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + assert.Nil(t, handler.ReelTimeout()) +} + +func TestCommand_ReelMatch(t *testing.T) { + tester, handlers, jsonParseResult, err := createTest() + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + // Positive Test + step := handler.ReelMatch(expectedPassPattern, "", "OK") + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, (*tester).Result()) +} diff --git a/pkg/tnf/handlers/command/doc.go b/pkg/tnf/handlers/command/doc.go new file mode 100644 index 000000000..bcc9cb36a --- /dev/null +++ b/pkg/tnf/handlers/command/doc.go @@ -0,0 +1,17 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package command provides a test for command. +package command diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 340e2f673..12a4c79e5 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -19,6 +19,7 @@ package identifier import "github.com/test-network-function/test-network-function/pkg/tnf/dependencies" const ( + commandIdentifierURL = "http://test-network-function.com/tests/command" nodeselectorIdentifierURL = "http://test-network-function.com/tests/nodeselector" hostnameIdentifierURL = "http://test-network-function.com/tests/hostname" ipAddrIdentifierURL = "http://test-network-function.com/tests/ipaddr" @@ -100,6 +101,16 @@ type IntrusionSettings struct { // Catalog is the test catalog. var Catalog = map[string]TestCatalogEntry{ + commandIdentifierURL: { + Identifier: CommandIdentifier, + Description: "A generic test used with any command and would match any output. The caller is responsible for interpreting the output and extracting data from it.", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{}, + }, hostnameIdentifierURL: { Identifier: HostnameIdentifier, Description: "A generic test used to check the hostname of a target machine/container.", @@ -611,6 +622,12 @@ var Catalog = map[string]TestCatalogEntry{ }, } +// CommandIdentifier is the Identifier used to represent the generic command test case. +var CommandIdentifier = Identifier{ + URL: commandIdentifierURL, + SemanticVersion: versionOne, +} + // HostnameIdentifier is the Identifier used to represent the generic hostname test case. var HostnameIdentifier = Identifier{ URL: hostnameIdentifierURL, From e4a985a5eb3987ce48eea38399501e4fbc39fae2 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 15 Oct 2021 08:57:16 +0200 Subject: [PATCH 093/344] CRD autodiscovery by nameSuffix. (#380) Added targetCrds yaml entry for CRDs autodiscovery. Currently, they can only be discovered using the nameSuffix tag, but it can be extended to use others like "group" or just labels as with pods/cuts. It just creates a list with the names of the CRDs that belong to the configured groups. Co-authored-by: Jun Chen --- README.md | 11 +++++ .../autodiscover/autodiscover_targets.go | 48 ++++++++++++++++++- pkg/config/config.go | 3 +- pkg/config/config_instance_test.go | 11 +++++ pkg/config/configsections/common.go | 2 + .../configsections/config_section_test.go | 34 ++++++------- pkg/config/configsections/crdfilter.go | 23 +++++++++ pkg/config/configsections/misc.go | 18 ------- pkg/config/testdata/tnf_test_config.yml | 3 ++ test-network-function/tnf_config.yml | 3 ++ 10 files changed, 115 insertions(+), 41 deletions(-) create mode 100644 pkg/config/configsections/crdfilter.go diff --git a/README.md b/README.md index a73a228ee..e96227905 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,17 @@ test-network-function.com/generic: target Once the pods are found, all of their containers are also added to the target container list. A target deployments list will also be created with all the deployments which the test pods belong to. +### targetCrds +In order to autodiscover the CRDs to be tested, an array of search filters can be set under the "targetCrdFilters" label. The autodiscovery mechanism will iterate through all the filters to look for all the CRDs that match it. Currently, filters only work by name suffix. + +```shell-script +targetCrdFilters: + - nameSuffix: "group1.tnf.com" + - nameSuffix: "anydomain.com" +``` + +The autodiscovery mechanism will create a list of all CRD names in the cluster whose names have the suffix "group1.tnf.com" or "anydomain.com", e.g. "crd1.group1.tnf.com" or "mycrd.mygroup.anydomain.com". + ### testTarget #### podsUnderTest / containersUnderTest This section is usually not required if labels defined in the section above cover all resources that should be tested. If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, instrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. The pods and containers explicitly configured here are added to the target pod/container lists populated through label matching. diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 569d5ddde..b61433e56 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -17,6 +17,10 @@ package autodiscover import ( + "encoding/json" + "os/exec" + "strings" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -27,8 +31,9 @@ import ( ) const ( - operatorLabelName = "operator" - skipConnectivityTestsLabel = "skip_connectivity_tests" + operatorLabelName = "operator" + skipConnectivityTestsLabel = "skip_connectivity_tests" + ocGetClusterCrdNamesCommand = "kubectl get crd -o json | jq '[.items[].metadata.name]'" ) var ( @@ -196,3 +201,42 @@ func getConfiguredOperatorTests() (opTests []string) { log.WithField("opTests", opTests).Infof("got all tests from %s.", testcases.ConfiguredTestFile) return opTests } + +// getClusterCrdNames returns a list of crd names found in the cluster. +func getClusterCrdNames() ([]string, error) { + // ToDo: Use command handler. + cmd := exec.Command("bash", "-c", ocGetClusterCrdNamesCommand) + + out, err := cmd.Output() + if err != nil { + return nil, err + } + + var crdNamesList []string + err = json.Unmarshal(out, &crdNamesList) + if err != nil { + return nil, err + } + + return crdNamesList, nil +} + +// FindTestCrdNames gets a list of CRD names based on configured groups. +func FindTestCrdNames(crdFilters []configsections.CrdFilter) []string { + clusterCrdNames, err := getClusterCrdNames() + if err != nil { + log.Errorf("Unable to get cluster CRD.") + return []string{} + } + + var targetCrdNames []string + for _, crdName := range clusterCrdNames { + for _, crdFilter := range crdFilters { + if strings.HasSuffix(crdName, crdFilter.NameSuffix) { + targetCrdNames = append(targetCrdNames, crdName) + break + } + } + } + return targetCrdNames +} diff --git a/pkg/config/config.go b/pkg/config/config.go index a655fbb81..3c6700201 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -127,7 +127,7 @@ type TestEnvironment struct { OperatorsUnderTest []configsections.Operator NameSpaceUnderTest string Nodes map[string]configsections.Node - + CrdNames []string // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -199,6 +199,7 @@ func (env *TestEnvironment) doAutodiscover() { env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest env.OperatorsUnderTest = env.Config.Operators env.Nodes = env.Config.Nodes + env.CrdNames = autodiscover.FindTestCrdNames(env.Config.CrdFilters) log.Infof("Test Configuration: %+v", *env) env.needsRefresh = false diff --git a/pkg/config/config_instance_test.go b/pkg/config/config_instance_test.go index a9efbd8f2..601e79856 100644 --- a/pkg/config/config_instance_test.go +++ b/pkg/config/config_instance_test.go @@ -32,6 +32,10 @@ const ( testDeploymentName = "test" testDeploymentNamespace = "default" testDeploymentReplicas = 2 + + testCrdsNumber = 2 + testCrdNameSuffix1 = "group1.test1.com" + testCrdNameSuffix2 = "test2.com" ) func testLoadedDeployments(t *testing.T, deployments []configsections.Deployment) { @@ -41,6 +45,12 @@ func testLoadedDeployments(t *testing.T, deployments []configsections.Deployment assert.Equal(t, deployments[0].Replicas, testDeploymentReplicas) } +func testLoadedCrds(t *testing.T, crds []configsections.CrdFilter) { + assert.Equal(t, len(crds), testCrdsNumber) + assert.Equal(t, crds[0].NameSuffix, testCrdNameSuffix1) + assert.Equal(t, crds[1].NameSuffix, testCrdNameSuffix2) +} + func TestLoadConfigFromFile(t *testing.T) { env := GetTestEnvironment() assert.Nil(t, env.loadConfigFromFile(filePath)) @@ -49,4 +59,5 @@ func TestLoadConfigFromFile(t *testing.T) { assert.Equal(t, env.Config.Partner.TestOrchestratorID.ContainerName, "partner") assert.Equal(t, env.Config.Partner.TestOrchestratorID.PodName, "partner") testLoadedDeployments(t, env.Config.DeploymentsUnderTest) + testLoadedCrds(t, env.Config.CrdFilters) } diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index f355d9fb7..31e792696 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -60,6 +60,8 @@ type TestConfiguration struct { CertifiedContainerInfo []CertifiedContainerRequestInfo `yaml:"certifiedcontainerinfo,omitempty" json:"certifiedcontainerinfo,omitempty"` // CertifiedOperatorInfo is list of operator bundle names that are queried for certification status. CertifiedOperatorInfo []CertifiedOperatorRequestInfo `yaml:"certifiedoperatorinfo,omitempty" json:"certifiedoperatorinfo,omitempty"` + // CRDs section. + CrdFilters []CrdFilter `yaml:"targetCrdFilters" json:"targetCrdFilters"` } // TestPartner contains the helper containers that can be used to facilitate tests diff --git a/pkg/config/configsections/config_section_test.go b/pkg/config/configsections/config_section_test.go index 9f15319da..b2805706f 100644 --- a/pkg/config/configsections/config_section_test.go +++ b/pkg/config/configsections/config_section_test.go @@ -41,19 +41,15 @@ const ( // cnfName name of the cnf cnfName = "cnf-test-one" // crdNameOne name of the crd - crdNameOne = "crd-test-one" + crdNameSuffix1 = "group1.test.com" // crdNameTwo name of the crd - crdNameTwo = "crd-test-two" + crdNameSuffix2 = "group2.test.com" // deploymentName is the name of the deployment deploymentName = "deployment-one" // deploymentReplicas no of replicas deploymentReplicas = 1 // fullConfig represents full configuration, including Operator and CNF fullConfig = "full_config" - // instanceNameOne name of the instance - instanceNameOne = "instance-one" - // instanceNameTwo name of the instance - instanceNameTwo = "instance-two" // operatorConfig represents operators configuration only operatorConfig = "operator_only_config" // operatorName name of the operator @@ -129,31 +125,23 @@ func loadOperatorConfig() { operator := Operator{} operator.Name = operatorName operator.Namespace = operatorNameSpace - setCrdsAndInstances() operator.Tests = []string{testcases.OperatorStatus} test.Operators = append(test.Operators, operator) loadPodConfig() } -func setCrdsAndInstances() { - crd := Crd{} - crd.Name = crdNameOne - crd.Namespace = testNameSpace - instance := Instance{} - instance.Name = instanceNameOne - crd.Instances = append(crd.Instances, instance) - crd2 := Crd{} - crd2.Name = crdNameTwo - crd2.Namespace = testNameSpace - instance2 := Instance{} - instance2.Name = instanceNameTwo - crd2.Instances = append(crd2.Instances, instance2) +func loadCrds() { + test.CrdFilters = []CrdFilter{ + {NameSuffix: crdNameSuffix1}, + {NameSuffix: crdNameSuffix2}, + } } func loadFullConfig() { loadOperatorConfig() loadPodConfig() loadDeploymentsConfig() + loadCrds() } func setup(configType string) { @@ -213,6 +201,10 @@ func TestFullConfigLoad(t *testing.T) { assert.NotNil(t, cfg) assert.Equal(t, len(cfg.Operators), 1) assert.Equal(t, cfg.PodsUnderTest[0].Name, cnfName) + + assert.Equal(t, cfg.CrdFilters[0].NameSuffix, crdNameSuffix1) + assert.Equal(t, cfg.CrdFilters[1].NameSuffix, crdNameSuffix2) + assert.Nil(t, err) } @@ -248,6 +240,8 @@ func TestFullJsonConfig(t *testing.T) { assert.NotNil(t, yamlCfg) assert.Equal(t, yamlCfg.Operators, jsonCfg.Operators) assert.Equal(t, yamlCfg.PodsUnderTest, jsonCfg.PodsUnderTest) + assert.Equal(t, yamlCfg.CrdFilters[0].NameSuffix, crdNameSuffix1) + assert.Equal(t, yamlCfg.CrdFilters[1].NameSuffix, crdNameSuffix2) } func TestCnfJsonConfig(t *testing.T) { diff --git a/pkg/config/configsections/crdfilter.go b/pkg/config/configsections/crdfilter.go new file mode 100644 index 000000000..9459b7a02 --- /dev/null +++ b/pkg/config/configsections/crdfilter.go @@ -0,0 +1,23 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +// CrdFilter defines a CustomResourceDefinition config filter. +type CrdFilter struct { + NameSuffix string `yaml:"nameSuffix" json:"nameSuffix"` + // labels []Label +} diff --git a/pkg/config/configsections/misc.go b/pkg/config/configsections/misc.go index 276116724..c67a95bd1 100644 --- a/pkg/config/configsections/misc.go +++ b/pkg/config/configsections/misc.go @@ -22,18 +22,6 @@ package configsections // CNFType defines a type to be either Operator or Container type CNFType string -// Crd struct defines Custom Resource Definition of the operator -type Crd struct { - // Name is the name of the CRD populated by the operator config generator - Name string `yaml:"name" json:"name"` - - // Namespace is the namespace where above CRD is installed(For all namespace this will be ALL_NAMESPACE) - Namespace string `yaml:"namespace" json:"namespace"` - - // Instances is the instance of CR matching for the above CRD KIND - Instances []Instance `yaml:"instances" json:"instances"` -} - // Permission defines roles and cluster roles resources type Permission struct { // Name is the name of Roles and Cluster Roles that is specified in the CSV @@ -42,9 +30,3 @@ type Permission struct { // Role is the role type either CLUSTER_ROLE or ROLE Role string `yaml:"role" json:"role"` } - -// Instance defines crd instances in the cluster -type Instance struct { - // Name is the name of the instance of custom resource (Auto populated) - Name string `yaml:"name" json:"name"` -} diff --git a/pkg/config/testdata/tnf_test_config.yml b/pkg/config/testdata/tnf_test_config.yml index 75ab788c1..b17c86fe5 100644 --- a/pkg/config/testdata/tnf_test_config.yml +++ b/pkg/config/testdata/tnf_test_config.yml @@ -47,3 +47,6 @@ certifiedcontainerinfo: certifiedoperatorinfo: - name: etcd-operator organization: redhat-marketplace +targetCrdFilters: + - nameSuffix: "group1.test1.com" + - nameSuffix: "test2.com" diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 7710d1d9c..7fa17b88e 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -4,6 +4,9 @@ targetPodLabels: - prefix: test-network-function.com name: generic value: target +targetCrdFilters: + - nameSuffix: "group1.test.com" + - nameSuffix: "example.com" # The following section does not require manual configuration as autodiscovery is on by default # Containers and pods will be found through matching targetPodLabels. Operators will be found if # labelled with "test-network-function.com/operator". Their subscription name will be read from From 533a33128e9908201a97cf949eb6007e7a59a7ff Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Mon, 18 Oct 2021 13:59:14 -0400 Subject: [PATCH 094/344] Invoke auto discovery for the diagnostic suite (#399) --- test-network-function/diagnostic/suite.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 4f33b49af..eeec737df 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -67,6 +67,12 @@ var ( var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.DiagnosticTestKey) { + env := config.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) + }) ginkgo.When("a cluster is set up and installed with OpenShift", func() { ginkgo.By("should report OCP version") From a2915255e173c0d535cf6f2b6431d4849c865961 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 19 Oct 2021 08:58:01 -0500 Subject: [PATCH 095/344] Fix URL for ping test (#400) --- CATALOG.md | 2 +- test-network-function/identifiers/identifiers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index c3f6a197f..6fbc7b779 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -157,7 +157,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. Result Type|normative -Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-nto-contain-ping-or-ip-binary-utilities). +Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities). ### http://test-network-function.com/testcases/networking/service-type Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index db8aa8e80..d2bf5524a 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -277,7 +277,7 @@ they are the same.`), CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: -[README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-nto-contain-ping-or-ip-binary-utilities).`, +[README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities).`, Description: formDescription(TestICMPv4ConnectivityIdentifier, `checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the From 9f291bee3abe8adff24b0705a276fdbcf69222b1 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Tue, 19 Oct 2021 17:10:03 +0200 Subject: [PATCH 096/344] CRD status handler using generic json template. (#391) Co-authored-by: Jun Chen --- CATALOG.md | 20 ++++ .../crdstatusexistence.json | 28 +++++ .../crdstatusexistence_test.go | 111 ++++++++++++++++++ pkg/tnf/handlers/crdstatusexistence/doc.go | 18 +++ pkg/tnf/identifier/identifiers.go | 20 ++++ 5 files changed, 197 insertions(+) create mode 100644 pkg/tnf/handlers/crdstatusexistence/crdstatusexistence.json create mode 100644 pkg/tnf/handlers/crdstatusexistence/crdstatusexistence_test.go create mode 100644 pkg/tnf/handlers/crdstatusexistence/doc.go diff --git a/CATALOG.md b/CATALOG.md index 6fbc7b779..e53519225 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -255,6 +255,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`oc` +### http://test-network-function.com/tests/command +Property|Description +---|--- +Version|v1.0.0 +Description|A generic test used with any command and would match any output. The caller is responsible for interpreting the output and extracting data from it. +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required| + ### http://test-network-function.com/tests/container/pod Property|Description ---|--- @@ -265,6 +275,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`jq`, `oc` +### http://test-network-function.com/tests/crdStatusExistence +Property|Description +---|--- +Version|v1.0.0 +Description|Checks whether a give CRD has status subresource specification. +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`oc`, `jq` + ### http://test-network-function.com/tests/csiDriver Property|Description ---|--- diff --git a/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence.json b/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence.json new file mode 100644 index 000000000..90065c428 --- /dev/null +++ b/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence.json @@ -0,0 +1,28 @@ +{ + "identifier": { + "url": "http://test-network-function.com/tests/crdStatusExistence", + "version": "v1.0.0" + }, + "description": "This test checks whether a given CRD has defined status subresource spec for all its versions.", + "testResult": 0, + "testTimeout": {{ .TIMEOUT }}, + "reelFirstStep": { + "execute": + "oc get crd {{ .CRD_NAME }} -o json | jq -r '[.spec.versions[]] | if all(.schema.openAPIV3Schema.properties.status) then \"OK\" else \"FAIL\" end'", + "expect": [ + "(?m)OK", + "(?m)FAIL" + ], + "timeout": {{ .TIMEOUT }} + }, + "resultContexts": [ + { + "pattern": "(?m)OK", + "defaultResult": 1 + }, + { + "pattern": "(?m)FAIL", + "defaultResult": 2 + } + ] + } diff --git a/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence_test.go b/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence_test.go new file mode 100644 index 000000000..0cf38554c --- /dev/null +++ b/pkg/tnf/handlers/crdstatusexistence/crdstatusexistence_test.go @@ -0,0 +1,111 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package crdstatusexistence_test + +import ( + "fmt" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/xeipuuv/gojsonschema" +) + +const ( + testTimeoutDuration = 10 * time.Second +) + +var ( + genericTestSchemaFile = path.Join("schemas", "generic-test.schema.json") + jsonTestFileName = "crdstatusexistence.json" + expectedPassPattern = "(?m)OK" + expectedFailPattern = "(?m)FAIL" + pathRelativeToRoot = path.Join("..", "..", "..", "..") + pathToTestSchemaFile = path.Join(pathRelativeToRoot, genericTestSchemaFile) + testCrdName = "testCrdFakeName" +) + +func createTest() (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { + values := make(map[string]interface{}) + values["CRD_NAME"] = testCrdName + values["TIMEOUT"] = testTimeoutDuration.Nanoseconds() + return generic.NewGenericFromMap(jsonTestFileName, pathToTestSchemaFile, values) +} + +func TestPods_Args(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Nil(t, (*test).Args()) +} + +func TestPods_GetIdentifier(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, identifier.CrdStatusExistenceIdentifier, (*test).GetIdentifier()) +} + +func TestPods_ReelFirst(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + step := handler.ReelFirst() + expectedReelFirstExecute := fmt.Sprintf( + "oc get crd %s -o json | jq -r '[.spec.versions[]] | if all(.schema.openAPIV3Schema.properties.status) then \"OK\" else \"FAIL\" end'", + testCrdName) + assert.Equal(t, expectedReelFirstExecute, step.Execute) + assert.Contains(t, step.Expect, expectedPassPattern, expectedFailPattern) + assert.Equal(t, testTimeoutDuration, step.Timeout) +} + +func TestPods_ReelMatch(t *testing.T) { + tester, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + + // Positive Test + step := handler.ReelMatch(expectedPassPattern, "", "OK") + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, (*tester).Result()) + + // Negative Test + step = handler.ReelMatch(expectedFailPattern, "", "FAIL") + assert.Nil(t, step) + assert.Equal(t, tnf.FAILURE, (*tester).Result()) +} diff --git a/pkg/tnf/handlers/crdstatusexistence/doc.go b/pkg/tnf/handlers/crdstatusexistence/doc.go new file mode 100644 index 000000000..8822126d4 --- /dev/null +++ b/pkg/tnf/handlers/crdstatusexistence/doc.go @@ -0,0 +1,18 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package crdstatusexistence provides a test for checking the existence of status subresource in a given CRD. +package crdstatusexistence diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 12a4c79e5..681da93c2 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -60,6 +60,7 @@ const ( scalingIdentifierURL = "http://test-network-function.com/tests/scaling" csiDriverIdentifierURL = "http://test-network-function.com/tests/csiDriver" clusterVersionIdentifierURL = "http://test-network-function.com/tests/clusterVersion" + crdStatusExistenceIdentifierURL = "http://test-network-function.com/tests/crdStatusExistence" versionOne = "v1.0.0" ) @@ -620,6 +621,19 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.OcBinaryName, }, }, + crdStatusExistenceIdentifierURL: { + Identifier: CrdStatusExistenceIdentifier, + Description: "Checks whether a give CRD has status subresource specification.", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + dependencies.JqBinaryName, + }, + }, } // CommandIdentifier is the Identifier used to represent the generic command test case. @@ -868,3 +882,9 @@ var ClusterVersionIdentifier = Identifier{ URL: clusterVersionIdentifierURL, SemanticVersion: versionOne, } + +// CrdStatusExistenceIdentifier is the Identifier used to represent the generic test for CRD status spec existence. +var CrdStatusExistenceIdentifier = Identifier{ + URL: crdStatusExistenceIdentifierURL, + SemanticVersion: versionOne, +} From ce4547ab7bfe08f155185c9ab7ae1f46d03b1910 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 20 Oct 2021 10:18:13 -0500 Subject: [PATCH 097/344] Get Release version from Git tag (#379) * Get Release version from Git tag * Adding support for building all currently released images (tag based) Co-authored-by: Jun Chen Co-authored-by: Salaheddine Hamadi --- .github/workflows/tnf-image.yaml | 62 +++++++++++++++++------------ Makefile | 8 ++-- script/create-version-files.sh | 10 +++++ script/get-git-latest-release.sh | 9 +++++ script/get-git-previous-release.sh | 5 +++ script/get-git-release.sh | 5 +++ test-network-function/suite_test.go | 35 +++++++++------- version.json | 1 - 8 files changed, 92 insertions(+), 43 deletions(-) create mode 100755 script/create-version-files.sh create mode 100755 script/get-git-latest-release.sh create mode 100755 script/get-git-previous-release.sh create mode 100755 script/get-git-release.sh diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 8e84c45ec..790619930 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -20,7 +20,6 @@ env: REGISTRY_LOCAL: localhost IMAGE_NAME: testnetworkfunction/test-network-function IMAGE_TAG: latest - CURRENT_VERSION_GENERIC_BRANCH: 3.0.x TNF_CONTAINER_CLIENT: docker TNF_MINIKUBE_ONLY: true TNF_NON_INTRUSIVE_ONLY: true @@ -33,46 +32,59 @@ env: TESTING_CMD_PARAMS: '-n host -i ${REGISTRY_LOCAL}/${IMAGE_NAME}:${IMAGE_TAG} -t ${TNF_CONFIG_DIR} -o ${TNF_OUTPUT_DIR}' jobs: - get-latest-release-version-numbers: - name: 'Get the version numbers of the latest release components' - if: ${{ github.repository_owner == 'test-network-function' }} + test-and-push-tnf-image: + name: 'Test and push the `test-network-function` image' runs-on: ubuntu-20.04 - outputs: - TNF_VERSION: ${{ steps.set_tnf_version.outputs.version_number }} - PARTNER_VERSION: ${{ steps.set_tnf_version.outputs.partner_version_number }} - + strategy: + matrix: + branch: [3.0.x] + env: + SHELL: /bin/bash + KUBECONFIG: '/home/runner/.kube/config' + CURRENT_VERSION_GENERIC_BRANCH: ${{ matrix.branch }} + TNF_VERSION: "" + PARTNER_VERSION: "" steps: + - name: Checkout generic working branch of the current version uses: actions/checkout@v2 with: ref: ${{ env.CURRENT_VERSION_GENERIC_BRANCH }} + fetch-depth: '0' + + - name: checkout the latest for the script directory in the main branch + run: | + git checkout origin/main -- script/create-version-files.sh + git checkout origin/main -- script/get-git-latest-release.sh + + - name: Get the latest TNF version from GIT + run: | + script/create-version-files.sh > /dev/null 2>&1 + echo "::set-output name=version_number::$(script/get-git-latest-release.sh)" + id: set_tnf_version + + - name: Print the latest TNF version from GIT + run: | + echo Version tag: ${{ steps.set_tnf_version.outputs.version_number }} - name: Get contents of the version.json file run: echo "::set-output name=json::$(cat version.json | tr -d '[:space:]')" id: get_version_json_file - - name: Save the version number to $TNF_VERSION + - name: Get the partner version number from file run: | - echo Version tag: $VERSION_FROM_FILE - echo "::set-output name=version_number::$VERSION_FROM_FILE" echo Partner version tag: $VERSION_FROM_FILE_PARTNER echo "::set-output name=partner_version_number::$VERSION_FROM_FILE_PARTNER" - id: set_tnf_version + id: set_partner_version env: - VERSION_FROM_FILE: ${{ fromJSON(steps.get_version_json_file.outputs.json).tag }} VERSION_FROM_FILE_PARTNER: ${{ fromJSON(steps.get_version_json_file.outputs.json).partner_tag }} - - test-and-push-tnf-image: - name: 'Test and push the `test-network-function` image' - needs: [get-latest-release-version-numbers] - runs-on: ubuntu-20.04 - env: - SHELL: /bin/bash - KUBECONFIG: '/home/runner/.kube/config' - TNF_VERSION: ${{ needs['get-latest-release-version-numbers'].outputs.TNF_VERSION }} - PARTNER_VERSION: ${{ needs['get-latest-release-version-numbers'].outputs.PARTNER_VERSION }} - - steps: + + - name: Update env variables + run: | + echo "TNF_VERSION=${{ steps.set_tnf_version.outputs.version_number }}" >> $GITHUB_ENV + echo "IMAGE_TAG=${{ steps.set_tnf_version.outputs.version_number }}" >> $GITHUB_ENV + echo "PARTNER_VERSION=${{ steps.set_partner_version.outputs.partner_version_number }}" >> $GITHUB_ENV + - name: Ensure $TNF_VERSION and $IMAGE_TAG are set run: '[[ -n "$TNF_VERSION" ]] && [[ -n "$IMAGE_TAG" ]] && [[ -n "$PARTNER_VERSION" ]]' diff --git a/Makefile b/Makefile index 710253c05..9084b7891 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,9 @@ else endif COMMON_GO_ARGS=-race -GIT_COMMIT=$(shell git rev-list -1 HEAD) +GIT_COMMIT=$(shell script/create-version-files.sh) +GIT_RELEASE=$(shell script/get-git-release.sh) +GIT_PREVIOUS_RELEASE=$(shell script/get-git-previous-release.sh) # Run the unit tests and build all binaries build: @@ -98,11 +100,11 @@ build-catalog-md: # build the CNF test binary build-cnf-tests: - PATH=${PATH}:${GOBIN} ginkgo build -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT}" ./test-network-function + PATH=${PATH}:${GOBIN} ginkgo build -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -X github.com/test-network-function/test-network-function/test-network-function.GitRelease=${GIT_RELEASE} -X github.com/test-network-function/test-network-function/test-network-function.GitPreviousRelease=${GIT_PREVIOUS_RELEASE}" ./test-network-function make build-catalog-md build-cnf-tests-debug: - PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -extldflags '-z relro -z now'" ./test-network-function + PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_RELEASE} -X github.com/test-network-function/test-network-function/test-network-function.GitPreviousRelease=${GIT_PREVIOUS_RELEASE} -extldflags '-z relro -z now'" ./test-network-function make build-catalog-md # run all CNF tests diff --git a/script/create-version-files.sh b/script/create-version-files.sh new file mode 100755 index 000000000..31783fac9 --- /dev/null +++ b/script/create-version-files.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +#set -x + +GIT_COMMIT=$(git rev-list -1 HEAD) + +curl -s https://api.github.com/repos/test-network-function/test-network-function/releases| jq -r ".[].tag_name"|sort > all-releases.txt +git tag --points-at HEAD |sort > release-tag.txt +git tag --no-contains ${GIT_COMMIT}|tail -n1|sort > latest-release-tag.txt + +echo $GIT_COMMIT diff --git a/script/get-git-latest-release.sh b/script/get-git-latest-release.sh new file mode 100755 index 000000000..d66814753 --- /dev/null +++ b/script/get-git-latest-release.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +#set -x +GIT_RELEASE=$(comm -12 all-releases.txt release-tag.txt) +GIT_PREVIOUS_RELEASE=$(comm -12 all-releases.txt latest-release-tag.txt) +GIT_LATEST_RELEASE=$GIT_RELEASE +if [ -z "$GIT_RELEASE" ]; then + GIT_LATEST_RELEASE=$GIT_PREVIOUS_RELEASE +fi +echo $GIT_LATEST_RELEASE \ No newline at end of file diff --git a/script/get-git-previous-release.sh b/script/get-git-previous-release.sh new file mode 100755 index 000000000..ee2d645ab --- /dev/null +++ b/script/get-git-previous-release.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#set -x + +GIT_PREVIOUS_RELEASE=$(comm -12 all-releases.txt latest-release-tag.txt) +echo $GIT_PREVIOUS_RELEASE \ No newline at end of file diff --git a/script/get-git-release.sh b/script/get-git-release.sh new file mode 100755 index 000000000..899c4a844 --- /dev/null +++ b/script/get-git-release.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#set -x + +GIT_RELEASE=$(comm -12 all-releases.txt release-tag.txt) +echo $GIT_RELEASE \ No newline at end of file diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 4c525c744..1df89ebcc 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -45,11 +45,9 @@ import ( _ "github.com/test-network-function/test-network-function/test-network-function/observability" _ "github.com/test-network-function/test-network-function/test-network-function/operator" _ "github.com/test-network-function/test-network-function/test-network-function/platform" - "github.com/test-network-function/test-network-function/test-network-function/version" ) const ( - programVersion = "3.0" claimFileName = "claim.json" claimFilePermissions = 0644 claimPathFlagKey = "claimloc" @@ -69,7 +67,17 @@ const ( var ( claimPath *string junitPath *string + // GitCommit is the latest commit in the current git branch GitCommit string + // GitRelease is the list of tags (if any) applied to the latest commit + // in the current branch + GitRelease string + // GitPreviousRelease is the last release at the date of the latest commit + // in the current branch + GitPreviousRelease string + // gitDisplayRelease is a string used to hold the text to display + // the version on screen and in the claim file + gitDisplayRelease string ) func init() { @@ -115,7 +123,16 @@ func TestTest(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) common.SetLogFormat() common.SetLogLevel() - log.Info("Version: ", programVersion, " ( ", GitCommit, " )") + + // Display the latest previously released build in case this build is not released + // Otherwise display the build version + if GitRelease == "" { + gitDisplayRelease = "Unreleased build post " + GitPreviousRelease + } else { + gitDisplayRelease = GitRelease + } + log.Info("Version: ", gitDisplayRelease, " ( ", GitCommit, " )") + tnfcommon.OcDebugImageID = common.GetOcDebugImageID() // Initialize the claim with the start time, tnf version, etc. @@ -152,20 +169,10 @@ func TestTest(t *testing.T) { writeClaimOutput(claimOutputFile, payload) } -// getTNFVersion gets the TNF version, or fatally fails. -func getTNFVersion() *version.Version { - // Extract the version, which should be placed by the build system. - tnfVersion, err := version.GetVersion() - if err != nil { - log.Fatalf("Couldn't determine the version: %v", err) - } - return tnfVersion -} - // incorporateTNFVersion adds the TNF version to the claim. func incorporateVersions(claimData *claim.Claim) { claimData.Versions = &claim.Versions{ - Tnf: getTNFVersion().Tag, + Tnf: gitDisplayRelease, TnfGitCommit: GitCommit, OcClient: diagnostic.GetVersionsOcp().Oc, Ocp: diagnostic.GetVersionsOcp().Ocp, diff --git a/version.json b/version.json index 086979156..201ddc59e 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,3 @@ { - "tag": "v3.0.0", "partner_tag": "v3.0.0" } From ebe029840e73e527d4bee6637c74021f4a80f4e2 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 20 Oct 2021 17:59:01 +0200 Subject: [PATCH 098/344] New TC: CRDs status subresource availability (#372) This will test the partner repo's CRD example in github ci actions. --- CATALOG.md | 8 ++++ .../identifiers/identifiers.go | 12 ++++++ test-network-function/observability/suite.go | 38 ++++++++++++++++++- test-network-function/tnf_config.yml | 2 +- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index e53519225..218763b60 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -70,6 +70,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| +### http://test-network-function.com/testcases/diagnostic/crd-status + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/diagnostic/crd-status checks that all CRDs have a status subresource specification. +Result Type|informative +Suggested Remediation|make sure that all the CRDs have a meaningful status specification. ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index d2bf5524a..674f29ab5 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -205,6 +205,11 @@ var ( Url: formTestURL(common.DiagnosticTestKey, "clusterversion"), Version: versionOne, } + // TestCrdsStatusSubresourceIdentifier ensures all CRDs have a valid status subresource + TestCrdsStatusSubresourceIdentifier = claim.Identifier{ + Url: formTestURL(common.DiagnosticTestKey, "crd-status"), + Version: versionOne, + } ) func formDescription(identifier claim.Identifier, description string) string { @@ -530,4 +535,11 @@ the changes for you.`, Description: formDescription(TestclusterVersionIdentifier, `Extracts OCP versions from the cluster.`), }, + TestCrdsStatusSubresourceIdentifier: { + Identifier: TestCrdsStatusSubresourceIdentifier, + Type: informativeResult, + Description: formDescription(TestCrdsStatusSubresourceIdentifier, + `checks that all CRDs have a status subresource specification.`), + Remediation: `make sure that all the CRDs have a meaningful status specification.`, + }, } diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index ec14ec2df..5d172e0a4 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -19,6 +19,7 @@ package observability import ( "fmt" "path" + "time" "github.com/onsi/ginkgo" ginkgoconfig "github.com/onsi/ginkgo/config" @@ -39,9 +40,15 @@ import ( var ( // loggingTestPath is the file location of the logging.json test case relative to the project root. loggingTestPath = path.Join("pkg", "tnf", "handlers", "logging", "logging.json") - // relativeLoggingTestPath is the relative path to the logging.json test case. relativeLoggingTestPath = path.Join(common.PathRelativeToRoot, loggingTestPath) + + // crdTestPath is the file location of the CRD status existence test case relative to the project root. + crdTestPath = path.Join("pkg", "tnf", "handlers", "crdstatusexistence", "crdstatusexistence.json") + // relativeCrdTestPath is the relatieve path to the crdstatusexistence.json test case. + relativeCrdTestPath = path.Join(common.PathRelativeToRoot, crdTestPath) + // testCrdsTimeout is the timeout in seconds for the CRDs TC. + testCrdsTimeout = 10 * time.Second ) var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { @@ -53,6 +60,7 @@ var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) testLogging(env) + testCrds(env) } }) @@ -86,3 +94,31 @@ func loggingTest(c configsections.ContainerIdentifier) { test.RunAndValidate() } + +func testCrds(env *config.TestEnvironment) { + ginkgo.By("CRDs should have a status subresource") + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestCrdsStatusSubresourceIdentifier) + ginkgo.It(testID, func() { + context := common.GetContext() + + for _, crdName := range env.CrdNames { + ginkgo.By("Testing CRD " + crdName) + + values := make(map[string]interface{}) + values["CRD_NAME"] = crdName + values["TIMEOUT"] = testCrdsTimeout.Nanoseconds() + + tester, handlers, result, err := generic.NewGenericFromMap(relativeCrdTestPath, common.RelativeSchemaPath, values) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) + gomega.Expect(test).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() + } + }) +} diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index 7fa17b88e..e8f0e139d 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -6,7 +6,7 @@ targetPodLabels: value: target targetCrdFilters: - nameSuffix: "group1.test.com" - - nameSuffix: "example.com" + - nameSuffix: "test-network-function.com" # The following section does not require manual configuration as autodiscovery is on by default # Containers and pods will be found through matching targetPodLabels. Operators will be found if # labelled with "test-network-function.com/operator". Their subscription name will be read from From a435ca2804d810131897156e50210a7aefdfaf3c Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 20 Oct 2021 13:13:43 -0500 Subject: [PATCH 099/344] removing no-effect incorect code to get rid of warnings (#403) --- .github/workflows/tnf-image.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 790619930..1ea7aaf46 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -123,13 +123,10 @@ jobs: with: repository: test-network-function/cnf-certification-test-partner path: cnf-certification-test-partner - with: ref: ${{ env.PARTNER_VERSION }} - name: Start the minikube cluster for `local-test-infra` uses: ./cnf-certification-test-partner/.github/actions/start-minikube - with: - working_directory: cnf-certification-test-partner - name: Create `local-test-infra` OpenShift resources uses: ./cnf-certification-test-partner/.github/actions/create-local-test-infra-resources From 8d1afa95e4d7eafbb639b45a51f7872fbc741aa5 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 20 Oct 2021 13:30:56 -0500 Subject: [PATCH 100/344] Upgrade to Go 1.17.2 (#402) * Upgrade to Go 1.17.2 * Add in adjustments to workflows from #395 Co-authored-by: Jun Chen --- .github/workflows/merge.yaml | 6 +++--- .github/workflows/pre-main.yaml | 20 ++++++++++---------- Dockerfile | 2 +- Makefile | 6 +++--- README.md | 2 +- go.mod | 20 +++++++++++++++++++- go.sum | 3 --- 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml index ab702b526..7b095f2cd 100644 --- a/.github/workflows/merge.yaml +++ b/.github/workflows/merge.yaml @@ -12,16 +12,16 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: ^1.16 + go-version: 1.17.2 - name: Check out code into the Go module directory uses: actions/checkout@v2 - name: Rebuild mocks - run: go install github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen@v1.6.0 && make mocks - name: Run Tests run: make test diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 557ad2f17..5c9167417 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -29,10 +29,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: ^1.16 + go-version: 1.17.2 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -40,7 +40,7 @@ jobs: ref: ${{ github.sha }} - name: Rebuild mocks - run: go install github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen@v1.6.0 && make mocks # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However # there does not appear to be a way to honor our existing .golangci.yml. For now, mimic developer behavior. @@ -77,10 +77,10 @@ jobs: runs-on: ubuntu-20.04 steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: ^1.16 + go-version: 1.17.2 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -88,7 +88,7 @@ jobs: ref: ${{ github.sha }} - name: Rebuild mocks - run: go install github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen@v1.6.0 && make mocks - name: Run Tests run: make test @@ -101,10 +101,10 @@ jobs: KUBECONFIG: '/home/runner/.kube/config' steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: ^1.16 + go-version: 1.17.2 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -112,10 +112,10 @@ jobs: ref: ${{ github.sha }} - name: Execute `make mocks` - run: go install github.com/golang/mock/mockgen && make mocks + run: go install github.com/golang/mock/mockgen@v1.6.0 && make mocks - name: Install ginkgo - run: go install github.com/onsi/ginkgo/ginkgo + run: go install github.com/onsi/ginkgo/ginkgo@v1.16.5 - name: Execute `make build` run: make build diff --git a/Dockerfile b/Dockerfile index 7ba566604..86e85ee2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN yum install -y gcc git jq make wget # Install Go binary ENV GO_DL_URL="https://golang.org/dl" -ENV GO_BIN_TAR="go1.16.8.linux-amd64.tar.gz" +ENV GO_BIN_TAR="go1.17.2.linux-amd64.tar.gz" ENV GO_BIN_URL_x86_64=${GO_DL_URL}/${GO_BIN_TAR} ENV GOPATH="/root/go" RUN if [[ "$(uname -m)" -eq "x86_64" ]] ; then \ diff --git a/Makefile b/Makefile index 9084b7891..5b69972da 100644 --- a/Makefile +++ b/Makefile @@ -148,9 +148,9 @@ update-deps: # Install build tools and other required software. install-tools: - go install github.com/onsi/ginkgo/ginkgo - go install github.com/onsi/gomega/... - go install github.com/golang/mock/mockgen + go install github.com/onsi/ginkgo/ginkgo@v1.16.5 + go install github.com/onsi/gomega + go install github.com/golang/mock/mockgen@v1.6.0 vet: go vet ${GO_PACKAGES} diff --git a/README.md b/README.md index e96227905..890512122 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ At a minimum, the following dependencies must be installed *prior* to running `m Dependency|Minimum Version ---|--- -[GoLang](https://golang.org/dl/)|1.16 +[GoLang](https://golang.org/dl/)|1.17 [golangci-lint](https://golangci-lint.run/usage/install/)|1.42.1 [jq](https://stedolan.github.io/jq/)|1.6 [OpenShift Client](https://docs.openshift.com/container-platform/4.4/welcome/index.html)|4.4 diff --git a/go.mod b/go.mod index 1643b179f..117407cb8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/test-network-function/test-network-function -go 1.16 +go 1.17 require ( github.com/Masterminds/semver/v3 v3.1.1 @@ -22,3 +22,21 @@ require ( google.golang.org/grpc v1.41.0 gopkg.in/yaml.v2 v2.4.0 ) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect + golang.org/x/text v0.3.6 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/go.sum b/go.sum index e9cf999a2..b49ac7cd0 100644 --- a/go.sum +++ b/go.sum @@ -85,7 +85,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -332,7 +331,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -512,7 +510,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8d11ed01c926801cbe63c5e732167bc2fc81a10b Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:30:14 -0500 Subject: [PATCH 101/344] Adding lint to list of tools installed by make (#404) --- .github/workflows/pre-main.yaml | 2 +- Dockerfile | 9 ++++----- Makefile | 8 +++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 5c9167417..5dbe4217f 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -45,7 +45,7 @@ jobs: # TODO: golangci-lint team recommends using a GitHub Action to perform golangci-lint responsibilities. However # there does not appear to be a way to honor our existing .golangci.yml. For now, mimic developer behavior. - name: Install golangci-lint - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1 + run: make install-lint - name: make lint run: make lint diff --git a/Dockerfile b/Dockerfile index 86e85ee2d..d6797658b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ ARG TNF_PARTNER_DIR=/usr/tnf-partner ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src -ENV GOLANGCI_VERSION=v1.42.1 ENV OPENSHIFT_VERSION=4.6.32 ENV TNF_DIR=/usr/tnf @@ -40,9 +39,6 @@ RUN wget --directory-prefix=${TEMP_DIR} ${OC_DL_URL} && \ # Add go and oc binary directory to $PATH ENV PATH=${PATH}:"/usr/local/go/bin":${GOPATH}/"bin" -# golangci-lint -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/bin ${GOLANGCI_VERSION} - # Git identifier to checkout ARG TNF_VERSION ARG TNF_SRC_URL=https://github.com/test-network-function/test-network-function @@ -53,7 +49,6 @@ ARG TNF_PARTNER_VERSION ARG TNF_PARTNER_SRC_URL=https://github.com/test-network-function/cnf-certification-test-partner ARG GIT_PARTNER_CHECKOUT_TARGET=$TNF_PARTNER_VERSION - # Clone the TNF source repository and checkout the target branch/tag/commit RUN git clone --no-single-branch --depth=1 ${TNF_SRC_URL} ${TNF_SRC_DIR} RUN git -C ${TNF_SRC_DIR} fetch origin ${GIT_CHECKOUT_TARGET} @@ -66,6 +61,10 @@ RUN git -C ${TNF_PARTNER_SRC_DIR} checkout ${GIT_PARTNER_CHECKOUT_TARGET} # Build TNF binary WORKDIR ${TNF_SRC_DIR} + +# golangci-lint +RUN make install-lint + # TODO: RUN make install-tools RUN make install-tools && \ make mocks && \ diff --git a/Makefile b/Makefile index 5b69972da..0164fec1a 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,9 @@ GO_PACKAGES=$(shell go list ./... | grep -v vendor) vet # Get default value of $GOBIN if not explicitly set +GO_PATH=$(shell go env GOPATH) ifeq (,$(shell go env GOBIN)) - GOBIN=$(shell go env GOPATH)/bin + GOBIN=${GO_PATH}/bin else GOBIN=$(shell go env GOBIN) endif @@ -49,6 +50,7 @@ COMMON_GO_ARGS=-race GIT_COMMIT=$(shell script/create-version-files.sh) GIT_RELEASE=$(shell script/get-git-release.sh) GIT_PREVIOUS_RELEASE=$(shell script/get-git-previous-release.sh) +GOLANGCI_VERSION=v1.42.1 # Run the unit tests and build all binaries build: @@ -152,5 +154,9 @@ install-tools: go install github.com/onsi/gomega go install github.com/golang/mock/mockgen@v1.6.0 +# Install golangci-lint +install-lint: + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GO_PATH}/bin ${GOLANGCI_VERSION} + vet: go vet ${GO_PACKAGES} From d0be7e047a8e0f13bf6d9305c2ff689ff847a573 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:43:43 -0500 Subject: [PATCH 102/344] Tainted kernel log improvement (#401) * Tainted kernel log improvement --- CATALOG.md | 2 +- pkg/tnf/handlers/nodetainted/nodetainted.go | 2 + pkg/tnf/identifier/identifiers.go | 5 +- test-network-function/platform/suite.go | 51 ++++++++++++++++++++ test-network-function/platform/suite_test.go | 28 +++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 test-network-function/platform/suite_test.go diff --git a/CATALOG.md b/CATALOG.md index 218763b60..3b6600767 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -528,7 +528,7 @@ Property|Description ---|--- Version|v1.0.0 Description|A generic test used to test whether node is tainted -Result Type|normative +Result Type|informative Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`oc`, `cat`, `echo` diff --git a/pkg/tnf/handlers/nodetainted/nodetainted.go b/pkg/tnf/handlers/nodetainted/nodetainted.go index 1151c429a..b11bd57c6 100644 --- a/pkg/tnf/handlers/nodetainted/nodetainted.go +++ b/pkg/tnf/handlers/nodetainted/nodetainted.go @@ -34,6 +34,7 @@ type NodeTainted struct { result int timeout time.Duration args []string + Match string } // NewNodeTainted creates a new NodeTainted tnf.Test. @@ -77,6 +78,7 @@ func (nt *NodeTainted) ReelFirst() *reel.Step { // ReelMatch tests whether node is tainted or not func (nt *NodeTainted) ReelMatch(_, _, match string) *reel.Step { + nt.Match = match if match == "0" { nt.result = tnf.SUCCESS } else { diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 681da93c2..d7e47cb97 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -67,7 +67,8 @@ const ( const ( // Normative is the test type used for a test that returns normative results. Normative = "normative" - // TODO: Informative = "informative" once we have informative tests. + // Informative is the test type used for a test that returns informative results. + Informative = "informative" ) // TestCatalogEntry is a container for required test facets. @@ -291,7 +292,7 @@ var Catalog = map[string]TestCatalogEntry{ nodeTaintedIdentifierURL: { Identifier: NodeTaintedIdentifier, Description: "A generic test used to test whether node is tainted", - Type: Normative, + Type: Informative, IntrusionSettings: IntrusionSettings{ ModifiesSystem: false, ModificationIsPersistent: false, diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index da3264fd9..ebec4e7d8 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -20,8 +20,11 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" "strings" + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" @@ -53,6 +56,29 @@ import ( // // All actual test code belongs below here. Utilities belong above. // + +func getTaintedBitValues() []string { + return []string{"proprietary module was loaded", + "module was force loaded", + "kernel running on an out of specification system", + "module was force unloaded", + "processor reported a Machine Check Exception (MCE)", + "bad page referenced or some unexpected page flags", + "taint requested by userspace application", + "kernel died recently, i.e. there was an OOPS or BUG", + "ACPI table overridden by user", + "kernel issued warning", + "staging driver was loaded", + "workaround for bug in platform firmware applied", + "externally-built (“out-of-tree”) module was loaded", + "unsigned module was loaded", + "soft lockup occurred", + "kernel has been live patched", + "auxiliary taint, defined for and used by distros", + "kernel was built with the struct randomization plugin", + } +} + var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.PlatformAlterationTestKey) { env := config.GetTestEnvironment() @@ -282,6 +308,18 @@ func testSysctlConfigsHelper(podName, podNamespace string) { } } +func printTainted(bitmap uint64) string { + values := getTaintedBitValues() + var out string + for i := 0; i < 32; i++ { + bit := (bitmap >> i) & 1 + if bit == 1 { + out += fmt.Sprintf("%s, ", values[i]) + } + } + return out +} + func testTainted(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonTaintedNodeKernelsIdentifier) ginkgo.It(testID, func() { @@ -296,6 +334,19 @@ func testTainted(env *config.TestEnvironment) { test.RunWithFailureCallback(func() { taintedNodes = append(taintedNodes, node.Name) }) + taintedBitmap, err := strconv.ParseUint(tester.Match, 10, 32) //nolint:gomnd // base 10 and uint32 + var message string + if err != nil { + message = fmt.Sprintf("Could not decode tainted kernel causes (code=%d) for node %s\n", taintedBitmap, node.Name) + } else if taintedBitmap != 0 { + message = fmt.Sprintf("Decoded tainted kernel causes (code=%d) for node %s : %s\n", taintedBitmap, node.Name, printTainted(taintedBitmap)) + } else { + message = fmt.Sprintf("Decoded tainted kernel causes (code=%d) for node %s : None\n", taintedBitmap, node.Name) + } + _, err = ginkgo.GinkgoWriter.Write([]byte(message)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } } gomega.Expect(taintedNodes).To(gomega.BeNil()) }) diff --git a/test-network-function/platform/suite_test.go b/test-network-function/platform/suite_test.go new file mode 100644 index 000000000..153c354bb --- /dev/null +++ b/test-network-function/platform/suite_test.go @@ -0,0 +1,28 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package platform + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_printTainted(t *testing.T) { + assert.Equal(t, printTainted(2048), "workaround for bug in platform firmware applied, ") + assert.Equal(t, printTainted(32769), "proprietary module was loaded, kernel has been live patched, ") +} From 8d55a810798eef6ed12e06a1ec6bec948aed1826 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 21 Oct 2021 20:45:04 +0200 Subject: [PATCH 103/344] Reel.go trace changed to debug level. (#406) --- pkg/tnf/reel/reel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tnf/reel/reel.go b/pkg/tnf/reel/reel.go index 3ebd89368..94cacbbd1 100644 --- a/pkg/tnf/reel/reel.go +++ b/pkg/tnf/reel/reel.go @@ -213,7 +213,7 @@ func (r *Reel) Step(step *Step, handler Handler) error { r.Err = fmt.Errorf("error executing command %d: ", outputStatus) } match, matchStatus := r.stripEmulatedPromptFromOutput(result.Match[0]) - log.Infof("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d", output, match, outputStatus, matchStatus) + log.Debugf("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d", output, match, outputStatus, matchStatus) matchIndex := strings.Index(output, match) var before string // special case: the match regex may be nothing at all. From fa4fa3c2b7b46ea96b73f763714e003edb727127 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Tue, 26 Oct 2021 07:59:02 -0500 Subject: [PATCH 104/344] Run nodeport test once per namspace (#409) --- test-network-function/networking/suite.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index e61ed98e4..4bd931b7a 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -135,15 +135,12 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count func testNodePort(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestServicesDoNotUseNodeportsIdentifier) ginkgo.It(testID, func() { - for _, podUnderTest := range env.PodsUnderTest { - defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) - context := common.GetContext() - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Testing services in namespace %s", podNamespace)) - tester := nodeport.NewNodePort(common.DefaultTimeout, podNamespace) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidate() - } + defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) + context := common.GetContext() + ginkgo.By(fmt.Sprintf("Testing services in namespace %s", env.NameSpaceUnderTest)) + tester := nodeport.NewNodePort(common.DefaultTimeout, env.NameSpaceUnderTest) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() }) } From 2faeb92e14e807c1114c882e5f9aba68c3434daa Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Tue, 26 Oct 2021 17:59:14 +0300 Subject: [PATCH 105/344] Integrate best practice reference into test catalog (#398) * Integrate best practice reference into test catalog * Update * Updated for every test the best practice reference. * Resolved comment of Gonzalo * resolved comments of Jun and Brandon * Resolved comment of Jun * Update resolved comment of Jun * Update resolved comment of Jun Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen --- CATALOG.md | 29 +++++++++++ cmd/catalog/cmd/generate.go | 1 + .../identifiers/identifiers.go | 50 ++++++++++++++++--- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 3b6600767..60ae80545 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -14,6 +14,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/cluster-role-bindings tests that a Pod does not specify ClusterRoleBindings. Result Type|normative Suggested Remediation|In most cases, Pod's should not have ClusterRoleBindings. The suggested remediation is to remove the need for ClusterRoleBindings, if possible. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.10 and 6.3.6 ### http://test-network-function.com/testcases/access-control/host-resource Property|Description @@ -22,6 +23,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/host-resource tests several aspects of CNF best practices, including: 1. The Pod does not have access to Host Node Networking. 2. The Pod does not have access to Host Node Ports. 3. The Pod cannot access Host Node IPC space. 4. The Pod cannot access Host Node PID space. 5. The Pod is not granted NET_ADMIN SCC. 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. Result Type|normative Suggested Remediation|Ensure that each Pod in the CNF abides by the suggested best practices listed in the test description. In some rare cases, not all best practices can be followed. For example, some CNFs may be required to run as root. Such exceptions should be handled on a case-by-case basis, and should provide a proper justification as to why the best practice(s) cannot be followed. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/namespace Property|Description @@ -30,6 +32,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/namespace tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace. Result Type|normative Suggested Remediation|Ensure that your CNF utilizes a CNF-specific namespace. Additionally, the CNF-specific namespace should not start with "openshift-", except in rare cases. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/pod-role-bindings Property|Description @@ -38,6 +41,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-role-bindings ensures that a CNF does not utilize RoleBinding(s) in a non-CNF Namespace. Result Type|normative Suggested Remediation|Ensure the CNF is not configured to use RoleBinding(s) in a non-CNF Namespace. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.5 ### http://test-network-function.com/testcases/access-control/pod-service-account Property|Description @@ -46,6 +50,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-service-account tests that each CNF Pod utilizes a valid Service Account. Result Type|normative Suggested Remediation|Ensure that the each CNF Pod is configured to use a valid Service Account +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.3 and 6.2.7 ### http://test-network-function.com/testcases/affiliated-certification/container-is-certified Property|Description @@ -54,6 +59,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/container-is-certified tests whether container images have passed the Red Hat Container Certification Program (CCP). Result Type|normative Suggested Remediation|Ensure that your container has passed the Red Hat Container Certification Program (CCP). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.7 ### http://test-network-function.com/testcases/affiliated-certification/operator-is-certified Property|Description @@ -62,6 +68,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/operator-is-certified tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP). Result Type|normative Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/diagnostic/clusterversion Property|Description @@ -70,6 +77,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/crd-status Property|Description @@ -78,6 +86,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/crd-status checks that all CRDs have a status subresource specification. Result Type|informative Suggested Remediation|make sure that all the CRDs have a meaningful status specification. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description @@ -86,6 +95,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/extract-node-information extracts informational information about the cluster. Result Type|informative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/list-cni-plugins Property|Description @@ -94,6 +104,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/list-cni-plugins lists CNI plugins Result Type|normative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.4 and 6.3.7 ### http://test-network-function.com/testcases/diagnostic/nodes-hw-info Property|Description @@ -102,6 +113,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/nodes-hw-info list nodes HW info Result Type|normative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/container-shutdown Property|Description @@ -110,6 +122,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/container-shutdown Ensure that the containers lifecycle pre-stop management feature is configured. Result Type|normative Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connexion). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace perdiod. 3) K8s will clean the remaining processes using KILL signal. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-high-availability Property|Description @@ -118,6 +131,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-high-availability ensures that CNF Pods specify podAntiAffinity rules and replica value is set to more than 1. Result Type|informative Suggested Remediation|In high availability cases, Pod podAntiAffinity rule should be specified for pod scheduling and pod replica value is set to more than 1 . +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-owner-type Property|Description @@ -126,6 +140,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s). Result Type|normative Suggested Remediation|Deploy the CNF using ReplicaSet/StatefulSet. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.8 ### http://test-network-function.com/testcases/lifecycle/pod-recreation Property|Description @@ -134,6 +149,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that a CNF is configured to support High Availability. First, this test cordons and drains a Node that hosts the CNF Pod. Next, the test ensures that OpenShift can re-instantiate the Pod on another Node, and that the actual replica count matches the desired replica count. Result Type|normative Suggested Remediation|Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-scheduling Property|Description @@ -142,6 +158,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-scheduling ensures that CNF Pods do not specify nodeSelector or nodeAffinity. In most cases, Pods should allow for instantiation on any underlying Node. Result Type|informative Suggested Remediation|In most cases, Pod's should not specify their host Nodes through nodeSelector or nodeAffinity. However, there are cases in which CNFs require specialized hardware specific to a particular class of Node. As such, this test is purely informative, and will not prevent a CNF from being certified. However, one should have an appropriate justification as to why nodeSelector and/or nodeAffinity is utilized by a CNF. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period Property|Description @@ -150,6 +167,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period tests whether the terminationGracePeriod is CNF-specific, or if the default (30s) is utilized. This test is informative, and will not affect CNF Certification. In many cases, the default terminationGracePeriod is perfectly acceptable for a CNF. Result Type|informative Suggested Remediation|Choose a terminationGracePeriod that is appropriate for your given CNF. If the default (30s) is appropriate, then feel free to ignore this informative message. This test is meant to raise awareness around how Pods are terminated, and to suggest that a CNF is configured based on its requirements. In addition to a terminationGracePeriod, consider utilizing a termination hook in the case that your application requires special shutdown instructions. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/scaling Property|Description @@ -158,6 +176,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/scaling tests that CNF deployments support scale in/out operations. First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s. Result Type|normative Suggested Remediation|Make sure CNF deployments/replica sets can scale in/out successfully. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/icmpv4-connectivity Property|Description @@ -166,6 +185,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. Result Type|normative Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/service-type Property|Description @@ -174,6 +194,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative Suggested Remediation|Ensure Services are not configured to use NodePort(s). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.1 ### http://test-network-function.com/testcases/operator/install-source Property|Description @@ -182,6 +203,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-source tests whether a CNF Operator is installed via OLM. Result Type|normative Suggested Remediation|Ensure that your Operator is installed via OLM. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/operator/install-status Property|Description @@ -190,6 +212,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. Result Type|normative Suggested Remediation|Ensure that your Operator abides by the Operator Best Practices mentioned in the description. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/platform-alteration/base-image Property|Description @@ -198,6 +221,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/base-image ensures that the Container Base Image is not altered post-startup. This test is a heuristic, and ensures that there are no changes to the following directories: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Result Type|normative Suggested Remediation|Ensure that Container applications do not modify the Container Base Image. In particular, ensure that the following directories are not modified: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Ensure that all required binaries are built directly into the container image, and are not installed post startup. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.2 ### http://test-network-function.com/testcases/platform-alteration/boot-params Property|Description @@ -206,6 +230,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/boot-params tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node. Result Type|normative Suggested Remediation|Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerformanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.13 and 6.2.14 ### http://test-network-function.com/testcases/platform-alteration/hugepages-config Property|Description @@ -214,6 +239,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/isredhat-release Property|Description @@ -222,6 +248,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release verifies if the container base image is redhat. Result Type|normative Suggested Remediation|build a new docker image that's based on UBI (redhat universal base image). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/sysctl-config Property|Description @@ -230,6 +257,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that no one has changed the node's sysctl configs after the node was created, the tests works by checking if the sysctl configs are consistent with the MachineConfig CR which defines how the node should be configured Result Type|normative Suggested Remediation|You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel Property|Description @@ -238,6 +266,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel ensures that the Node(s) hosting CNFs do not utilize tainted kernels. This test case is especially important to support Highly Available CNFs, since when a CNF is re-instantiated on a backup Node, that Node's kernel may not have the same hacks.' Result Type|normative Suggested Remediation|Test failure indicates that the underlying Node's' kernel is tainted. Ensure that you have not altered underlying Node(s) kernels in order to run the CNF. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.14 ## Test Case Building Blocks Catalog diff --git a/cmd/catalog/cmd/generate.go b/cmd/catalog/cmd/generate.go index 80a14075c..348354032 100644 --- a/cmd/catalog/cmd/generate.go +++ b/cmd/catalog/cmd/generate.go @@ -145,6 +145,7 @@ func outputTestCases() { fmt.Fprintf(os.Stdout, "Description|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Description, "\n", " ")) fmt.Fprintf(os.Stdout, "Result Type|%s\n", identifiers.Catalog[k].Type) fmt.Fprintf(os.Stdout, "Suggested Remediation|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Remediation, "\n", " ")) + fmt.Fprintf(os.Stdout, "Best Practice Reference|%s\n", strings.ReplaceAll(identifiers.Catalog[k].BestPracticeReference, "\n", " ")) } fmt.Println() fmt.Println() diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 674f29ab5..06e371430 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -26,10 +26,11 @@ import ( ) const ( - informativeResult = "informative" - normativeResult = "normative" - url = "http://test-network-function.com/testcases" - versionOne = "v1.0.0" + bestPracticeDocV1dot2URL = "[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf)" + informativeResult = "informative" + normativeResult = "normative" + url = "http://test-network-function.com/testcases" + versionOne = "v1.0.0" ) // TestCaseDescription describes a JUnit test case. @@ -45,6 +46,9 @@ type TestCaseDescription struct { // Type is the type of the test (i.e., normative). Type string `json:"type" yaml:"type"` + + // BestPracticeReference is a helpful best practice references of the test case. + BestPracticeReference string `json:"BestPracticeReference" yaml:"BestPracticeReference"` } func formTestURL(suite, name string) string { @@ -243,6 +247,7 @@ cannot be followed.`, 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. `), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestContainerIsCertifiedIdentifier: { @@ -251,6 +256,7 @@ cannot be followed.`, Remediation: `Ensure that your container has passed the Red Hat Container Certification Program (CCP).`, Description: formDescription(TestContainerIsCertifiedIdentifier, `tests whether container images have passed the Red Hat Container Certification Program (CCP).`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.7", }, TestExtractNodeInformationIdentifier: { @@ -258,6 +264,7 @@ cannot be followed.`, Type: informativeResult, Description: formDescription(TestExtractNodeInformationIdentifier, `extracts informational information about the cluster.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.6", }, TestHugepagesNotManuallyManipulated: { @@ -273,6 +280,7 @@ underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestICMPv4ConnectivityIdentifier: { @@ -289,6 +297,7 @@ test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. `), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestNamespaceBestPracticesIdentifier: { @@ -300,6 +309,7 @@ should not start with "openshift-", except in rare cases.`, `tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestNonDefaultGracePeriodIdentifier: { @@ -313,6 +323,7 @@ a termination hook in the case that your application requires special shutdown i `tests whether the terminationGracePeriod is CNF-specific, or if the default (30s) is utilized. This test is informative, and will not affect CNF Certification. In many cases, the default terminationGracePeriod is perfectly acceptable for a CNF.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestNonTaintedNodeKernelsIdentifier: { @@ -324,6 +335,7 @@ Node(s) kernels in order to run the CNF.`, `ensures that the Node(s) hosting CNFs do not utilize tainted kernels. This test case is especially important to support Highly Available CNFs, since when a CNF is re-instantiated on a backup Node, that Node's kernel may not have the same hacks.'`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.14", }, TestOperatorInstallStatusIdentifier: { @@ -335,6 +347,7 @@ the same hacks.'`), 1. The Operator CSV reports "Installed" status. 2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.12 and Section 6.3.3", }, TestOperatorIsCertifiedIdentifier: { @@ -343,6 +356,7 @@ with no resourceNames under its rules.`), Remediation: `Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP).`, Description: formDescription(TestOperatorIsCertifiedIdentifier, `tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP).`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.12 and Section 6.3.3", }, TestOperatorIsInstalledViaOLMIdentifier: { @@ -351,6 +365,7 @@ with no resourceNames under its rules.`), Remediation: `Ensure that your Operator is installed via OLM.`, Description: formDescription(TestOperatorIsInstalledViaOLMIdentifier, `tests whether a CNF Operator is installed via OLM.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.12 and Section 6.3.3", }, TestPodNodeSelectorAndAffinityBestPractices: { @@ -363,6 +378,7 @@ to why nodeSelector and/or nodeAffinity is utilized by a CNF.`, Description: formDescription(TestPodNodeSelectorAndAffinityBestPractices, `ensures that CNF Pods do not specify nodeSelector or nodeAffinity. In most cases, Pods should allow for instantiation on any underlying Node.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestPodHighAvailabilityBestPractices: { @@ -371,6 +387,7 @@ instantiation on any underlying Node.`), Remediation: `In high availability cases, Pod podAntiAffinity rule should be specified for pod scheduling and pod replica value is set to more than 1 .`, Description: formDescription(TestPodHighAvailabilityBestPractices, `ensures that CNF Pods specify podAntiAffinity rules and replica value is set to more than 1.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestPodClusterRoleBindingsBestPracticesIdentifier: { @@ -380,6 +397,7 @@ instantiation on any underlying Node.`), ClusterRoleBindings, if possible.`, Description: formDescription(TestPodClusterRoleBindingsBestPracticesIdentifier, `tests that a Pod does not specify ClusterRoleBindings.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.10 and 6.3.6", }, TestPodDeploymentBestPracticesIdentifier: { @@ -388,6 +406,7 @@ ClusterRoleBindings, if possible.`, Remediation: `Deploy the CNF using ReplicaSet/StatefulSet.`, Description: formDescription(TestPodDeploymentBestPracticesIdentifier, `tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s).`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.3 and 6.3.8", }, TestPodRoleBindingsBestPracticesIdentifier: { @@ -396,6 +415,7 @@ ClusterRoleBindings, if possible.`, Remediation: `Ensure the CNF is not configured to use RoleBinding(s) in a non-CNF Namespace.`, Description: formDescription(TestPodRoleBindingsBestPracticesIdentifier, `ensures that a CNF does not utilize RoleBinding(s) in a non-CNF Namespace.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.3 and 6.3.5", }, TestPodServiceAccountBestPracticesIdentifier: { @@ -404,6 +424,7 @@ ClusterRoleBindings, if possible.`, Remediation: `Ensure that the each CNF Pod is configured to use a valid Service Account`, Description: formDescription(TestPodServiceAccountBestPracticesIdentifier, `tests that each CNF Pod utilizes a valid Service Account.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.3 and 6.2.7", }, TestServicesDoNotUseNodeportsIdentifier: { @@ -412,6 +433,7 @@ ClusterRoleBindings, if possible.`, Remediation: `Ensure Services are not configured to use NodePort(s).`, Description: formDescription(TestServicesDoNotUseNodeportsIdentifier, `tests that each CNF Service does not utilize NodePort(s).`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.1", }, TestUnalteredBaseImageIdentifier: { @@ -443,6 +465,7 @@ that there are no changes to the following directories: 8) /usr/sbin 9) /usr/lib 10) /usr/lib64`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.2", }, TestUnalteredStartupBootParamsIdentifier: { @@ -452,6 +475,7 @@ that there are no changes to the following directories: the changes for you.`, Description: formDescription(TestUnalteredStartupBootParamsIdentifier, `tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.13 and 6.2.14", }, TestListCniPluginsIdentifier: { Identifier: TestListCniPluginsIdentifier, @@ -459,6 +483,7 @@ the changes for you.`, Remediation: "", Description: formDescription(TestListCniPluginsIdentifier, `lists CNI plugins`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2.4 and 6.3.7", }, TestNodesHwInfoIdentifier: { Identifier: TestNodesHwInfoIdentifier, @@ -466,6 +491,7 @@ the changes for you.`, Remediation: "", Description: formDescription(TestNodesHwInfoIdentifier, `list nodes HW info`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestShudtownIdentifier: { @@ -486,6 +512,7 @@ the changes for you.`, 2) K8s will wait for a grace perdiod. 3) K8s will clean the remaining processes using KILL signal. `, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestPodRecreationIdentifier: { Identifier: TestPodRecreationIdentifier, @@ -497,6 +524,7 @@ the changes for you.`, and that the actual replica count matches the desired replica count.`), Remediation: `Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails.`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestSysctlConfigsIdentifier: { Identifier: TestSysctlConfigsIdentifier, @@ -505,7 +533,8 @@ the changes for you.`, `tests that no one has changed the node's sysctl configs after the node was created, the tests works by checking if the sysctl configs are consistent with the MachineConfig CR which defines how the node should be configured`), - Remediation: `You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes`, + Remediation: `You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestScalingIdentifier: { Identifier: TestScalingIdentifier, @@ -514,32 +543,37 @@ the changes for you.`, `tests that CNF deployments support scale in/out operations. First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s.`), - Remediation: `Make sure CNF deployments/replica sets can scale in/out successfully.`, + Remediation: `Make sure CNF deployments/replica sets can scale in/out successfully.`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestIsRedHatReleaseIdentifier: { Identifier: TestIsRedHatReleaseIdentifier, Type: normativeResult, Description: formDescription(TestIsRedHatReleaseIdentifier, `verifies if the container base image is redhat.`), - Remediation: `build a new docker image that's based on UBI (redhat universal base image).`, + Remediation: `build a new docker image that's based on UBI (redhat universal base image).`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, TestClusterCsiInfoIdentifier: { Identifier: TestClusterCsiInfoIdentifier, Type: informativeResult, Description: formDescription(TestClusterCsiInfoIdentifier, `extracts CSI driver information in the cluster.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.6", }, TestClusterCsiInfoIdentifier: { Identifier: TestclusterVersionIdentifier, Type: informativeResult, Description: formDescription(TestclusterVersionIdentifier, `Extracts OCP versions from the cluster.`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.6", }, TestCrdsStatusSubresourceIdentifier: { Identifier: TestCrdsStatusSubresourceIdentifier, Type: informativeResult, Description: formDescription(TestCrdsStatusSubresourceIdentifier, `checks that all CRDs have a status subresource specification.`), - Remediation: `make sure that all the CRDs have a meaningful status specification.`, + Remediation: `make sure that all the CRDs have a meaningful status specification.`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, } From 03b5d5c464d40279ab8bcb09ad20ab72014a056f Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Tue, 26 Oct 2021 12:34:09 -0400 Subject: [PATCH 106/344] Replacing adhoc oc debug commands with precreated shell sessions to the debug pods managed by a daemon set (#395) * open debug session with all nodes kill pods instead of exit command remove dependency to common from packages fix linter fix mocks and github workflow * address PR comments use daemon set revert changes to github workflow add chroot when required * fix lint * fix lint * fix minikube issue * fix Jun's comments * run go mod tidy * specify mockgen version to fix github action * stop looking for debug pods in minikube * Update go.sum * Revert "specify mockgen version to fix github action" This reverts commit 4744426ce08576f4297e2031e5b04fef8bb9f9f3. * Fix make install-tools * Revert "Revert "specify mockgen version to fix github action"" This reverts commit 34686298381d5af2a0fb8c63cbdb53ea818824e6. * use new label for debug pods * add node selector to pod debug * add daemonset handler to check the ds availability add common suite for common setup and teardown * update label to match partner repo * update label * add label to node --- CATALOG.md | 10 ++ pkg/config/autodiscover/autodiscover.go | 49 ++++-- pkg/config/autodiscover/autodiscover_debug.go | 114 +++++++++++++ .../autodiscover/autodiscover_partner.go | 8 + .../autodiscover/autodiscover_targets.go | 11 +- pkg/config/config.go | 142 +++++++++++++++- pkg/config/configsections/common.go | 2 + .../bootconfigentries/bootconfigentries.go | 5 +- .../bootconfigentries_test.go | 9 +- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 7 +- pkg/tnf/handlers/command/command.json | 6 +- pkg/tnf/handlers/command/command_test.go | 2 +- pkg/tnf/handlers/common/command.go | 20 ++- pkg/tnf/handlers/daemonset/daemonset.go | 159 ++++++++++++++++++ pkg/tnf/handlers/daemonset/daemonset_test.go | 121 +++++++++++++ pkg/tnf/handlers/daemonset/doc.go | 17 ++ .../daemonset/testdata/non_valid_daemonset | 2 + .../daemonset/testdata/non_valid_output | 1 + .../daemonset/testdata/valid_daemonset | 2 + .../deploymentsdrain/deploymentsdrain.go | 2 +- pkg/tnf/handlers/nodedebug/nodedebug.go | 3 +- .../handlers/nodehugepages/nodehugepages.go | 5 +- .../nodehugepages/nodehugepages_test.go | 13 +- pkg/tnf/handlers/nodetainted/nodetainted.go | 5 +- .../handlers/nodetainted/nodetainted_test.go | 15 +- .../handlers/readbootconfig/readbootconfig.go | 6 +- .../readbootconfig/readbootconfig_test.go | 10 +- .../sysctlallconfigsargs.go | 5 +- .../sysctlallconfigsargs_test.go | 9 +- pkg/tnf/identifier/identifiers.go | 18 ++ pkg/tnf/interactive/oc.go | 23 ++- pkg/tnf/interactive/shell.go | 14 ++ test-network-function/common/constant.go | 1 + test-network-function/common/suite.go | 45 +++++ test-network-function/diagnostic/suite.go | 32 ++-- test-network-function/lifecycle/doc.go | 2 +- test-network-function/lifecycle/suite.go | 63 ++----- test-network-function/platform/suite.go | 41 ++--- 38 files changed, 824 insertions(+), 175 deletions(-) create mode 100644 pkg/config/autodiscover/autodiscover_debug.go create mode 100644 pkg/tnf/handlers/daemonset/daemonset.go create mode 100644 pkg/tnf/handlers/daemonset/daemonset_test.go create mode 100644 pkg/tnf/handlers/daemonset/doc.go create mode 100644 pkg/tnf/handlers/daemonset/testdata/non_valid_daemonset create mode 100644 pkg/tnf/handlers/daemonset/testdata/non_valid_output create mode 100644 pkg/tnf/handlers/daemonset/testdata/valid_daemonset create mode 100644 test-network-function/common/suite.go diff --git a/CATALOG.md b/CATALOG.md index 60ae80545..abed97e1a 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -342,6 +342,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`cat` +### http://test-network-function.com/tests/daemonset +Property|Description +---|--- +Version|v1.0.0 +Description|check whether a given daemonset was deployed successfully +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`oc` + ### http://test-network-function.com/tests/deployments Property|Description ---|--- diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 64291374f..7e6c38829 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -28,7 +28,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" - "github.com/test-network-function/test-network-function/test-network-function/common" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" ) const ( @@ -42,12 +42,16 @@ const ( ) var ( + // PathRelativeToRoot is used to calculate relative filepaths for the `test-network-function` executable entrypoint. + pathRelativeToRoot = path.Join("..") // TestFile is the file location of the command.json test case relative to the project root. TestFile = path.Join("pkg", "tnf", "handlers", "command", "command.json") - + // RelativeSchemaPath is the relative path to the generic-test.schema.json JSON schema. + relativeSchemaPath = path.Join(pathRelativeToRoot, schemaPath) // pathToTestFile is the relative path to the command.json test case. - pathToTestFile = path.Join(common.PathRelativeToRoot, TestFile) - + pathToTestFile = path.Join(pathRelativeToRoot, TestFile) + // schemaPath is the path to the generic-test.schema.json JSON schema relative to the project root. + schemaPath = path.Join("schemas", "generic-test.schema.json") // commandDriver stores the csi driver JSON output. commandDriver = make(map[string]interface{}) ) @@ -78,36 +82,45 @@ func buildLabelQuery(label configsections.Label) string { } func executeOcGetCommand(resourceType, labelQuery, namespace string) (string, error) { - ocCommandtoExecute := fmt.Sprintf(ocCommand, resourceType, namespace, labelQuery) + ocCommandToExecute := fmt.Sprintf(ocCommand, resourceType, namespace, labelQuery) + match, err := executeOcCommand(ocCommandToExecute) + if err != nil { + log.Error("can't run command, ", ocCommandToExecute, "Error=", err) + return "", err + } + err = jsonUnmarshal([]byte(match), &commandDriver) + gomega.Expect(err).To(gomega.BeNil()) + return match, err +} + +func executeOcCommand(command string) (string, error) { values := make(map[string]interface{}) - values["COMMAND"] = ocCommandtoExecute + values["COMMAND"] = command values["TIMEOUT"] = ocCommandTimeOut.Nanoseconds() - context := common.GetContext() - test, handler, result, err := generic.NewGenericFromMap(pathToTestFile, common.RelativeSchemaPath, values) + context := interactive.GetContext() + tester, handler, result, err := generic.NewGenericFromMap(pathToTestFile, relativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) gomega.Expect(result.Valid()).To(gomega.BeTrue()) gomega.Expect(handler).ToNot(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - - tester, err := tnf.NewTest(context.GetExpecter(), *test, handler, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) - testResult, err := tester.Run() + test, err := tnf.NewTest(context.GetExpecter(), *tester, handler, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(tester).ToNot(gomega.BeNil()) + if err != nil { + return "", err + } + test.RunAndValidate() - genericTest := (*test).(*generic.Generic) + genericTest := (*tester).(*generic.Generic) gomega.Expect(genericTest).ToNot(gomega.BeNil()) matches := genericTest.Matches gomega.Expect(len(matches)).To(gomega.Equal(1)) match := genericTest.GetMatches()[0] - err = jsonUnmarshal([]byte(match.Match), &commandDriver) - gomega.Expect(err).To(gomega.BeNil()) - return match.Match, err + return match.Match, nil } // getContainersByLabel builds `config.Container`s from containers in pods matching a label. diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go new file mode 100644 index 000000000..e09127f05 --- /dev/null +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -0,0 +1,114 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "fmt" + "time" + + "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf" + ds "github.com/test-network-function/test-network-function/pkg/tnf/handlers/daemonset" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +const ( + defaultNamespace = "default" + debugDaemonSet = "debug" + debugLabelName = "test-network-function.com/app" + debugLabelValue = "debug" + nodeLabelName = "test-network-function.com/node" + nodeLabelValue = "target" + addlabelCommand = "oc label node %s %s=%s --overwrite=true" + deletelabelCommand = "oc label node %s %s- --overwrite=true" +) + +// FindDebugPods completes a `configsections.TestPartner.ContainersDebugList` from the current state of the cluster, +// using labels and annotations to populate the data, if it's not fully configured +func FindDebugPods(tp *configsections.TestPartner) { + label := configsections.Label{Name: debugLabelName, Value: debugLabelValue} + pods, err := GetPodsByLabel(label, defaultNamespace) + if err != nil { + log.Panic("can't find debug pods") + } + if len(pods.Items) == 0 { + log.Panic("can't find debug pods, make sure daemonset debug is deployed properly") + } + for _, pod := range pods.Items { + tp.ContainersDebugList = append(tp.ContainersDebugList, buildContainersFromPodResource(pod)[0]) + } +} + +// AddDebugLabel add debug label to node +func AddDebugLabel(nodeName string) { + log.Info("add label", nodeLabelName, "=", nodeLabelValue, " to node ", nodeName) + ocCommand := fmt.Sprintf(addlabelCommand, nodeName, nodeLabelName, nodeLabelValue) + _, err := executeOcCommand(ocCommand) + if err != nil { + log.Error("error in adding label to node ", nodeName) + return + } +} + +// AddDebugLabel remove debug label from node +func DeleteDebugLabel(nodeName string) { + log.Info("delete label", nodeLabelName, "=", nodeLabelValue, "to node ", nodeName) + ocCommand := fmt.Sprintf(deletelabelCommand, nodeName, nodeLabelName) + _, err := executeOcCommand(ocCommand) + if err != nil { + log.Error("error in removing label from node ", nodeName) + return + } +} + +// CheckDebugDaemonset checks if the debug pods are deployed properly +// the function will try DefaultTimeout/time.Second times +func CheckDebugDaemonset() { + gomega.Eventually(func() bool { + log.Debug("check debug daemonset status") + return checkDebugPodsReadiness() + }, DefaultTimeout, 2*time.Second).Should(gomega.Equal(true)) //nolint: gomnd +} + +// checkDebugPodsReadiness helper function that returns true if the daemonset debug is deployed properly +func checkDebugPodsReadiness() bool { + context := interactive.GetContext() + tester := ds.NewDaemonSet(DefaultTimeout, debugDaemonSet, defaultNamespace) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + if err != nil { + log.Error("can't run test to detect daemonset status") + return false + } + _, err = test.Run() + if err != nil { + return false + } + dsStatus := tester.GetStatus() + if dsStatus.Desired == dsStatus.Current && + dsStatus.Available == dsStatus.Ready && + dsStatus.Ready == dsStatus.Desired && + dsStatus.Ready != 0 && + dsStatus.Misscheduled == 0 { + log.Info("daemonset is ready") + return true + } + log.Warn("daemonset is not ready") + return false +} diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index 04be8de75..16c1556f3 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -17,6 +17,9 @@ package autodiscover import ( + "os" + "strconv" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" ) @@ -26,6 +29,11 @@ const ( orchestratorValue = "orchestrator" ) +func IsMinikube() bool { + minikube, _ := strconv.ParseBool(os.Getenv("TNF_MINIKUBE_ONLY")) + return minikube +} + // FindTestPartner completes a `configsections.TestPartner` from the current state of the cluster, // using labels and annotations to populate the data, if it's not fully configured func FindTestPartner(tp *configsections.TestPartner, namespace string) { diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index b61433e56..2d879529e 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -20,20 +20,22 @@ import ( "encoding/json" "os/exec" "strings" + "time" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodenames" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" - "github.com/test-network-function/test-network-function/test-network-function/common" ) const ( operatorLabelName = "operator" skipConnectivityTestsLabel = "skip_connectivity_tests" ocGetClusterCrdNamesCommand = "kubectl get crd -o json | jq '[.items[].metadata.name]'" + DefaultTimeout = 10 * time.Second ) var ( @@ -82,8 +84,8 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa func GetNodesList() (nodes map[string]configsections.Node) { nodes = make(map[string]configsections.Node) var nodeNames []string - context := common.GetContext() - tester := nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{configsections.MasterLabel: nil}) + context := interactive.GetContext() + tester := nodenames.NewNodeNames(DefaultTimeout, map[string]*string{configsections.MasterLabel: nil}) test, _ := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) _, err := test.Run() if err != nil { @@ -98,8 +100,7 @@ func GetNodesList() (nodes map[string]configsections.Node) { } } - context = common.GetContext() - tester = nodenames.NewNodeNames(common.DefaultTimeout, map[string]*string{configsections.WorkerLabel: nil}) + tester = nodenames.NewNodeNames(DefaultTimeout, map[string]*string{configsections.WorkerLabel: nil}) test, _ = tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) _, err = test.Run() if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 3c6700201..0b1bc9c78 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -63,6 +63,33 @@ type Container struct { ContainerIdentifier configsections.ContainerIdentifier } +type NodeConfig struct { + // same Name as the one inside Node structure + Name string + Node configsections.Node + // Oc holds shell for debug pod running on the node + Oc *interactive.Oc + // deployment indicates if the node has a deployment + deployment bool + // debug indicates if the node should have a debug pod + debug bool +} + +func (n NodeConfig) IsMaster() bool { + return n.Node.IsMaster() +} + +func (n NodeConfig) IsWorker() bool { + return n.Node.IsWorker() +} + +func (n NodeConfig) HasDeployment() bool { + return n.deployment +} +func (n NodeConfig) HasDebugPod() bool { + return n.debug +} + // DefaultTimeout for creating new interactive sessions (oc, ssh, tty) var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second @@ -87,9 +114,9 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio select { case err := <-outCh: log.Fatalf("OC session to container %s/%s is broken due to: %v, aborting the test run", oc.GetPodName(), oc.GetPodContainerName(), err) - os.Exit(1) case <-oc.GetDoneChannel(): - break + log.Infof("stop watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) + return } } }() @@ -122,12 +149,14 @@ func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) (string type TestEnvironment struct { ContainersUnderTest map[configsections.ContainerIdentifier]*Container PartnerContainers map[configsections.ContainerIdentifier]*Container + DebugContainers map[configsections.ContainerIdentifier]*Container PodsUnderTest []configsections.Pod DeploymentsUnderTest []configsections.Deployment OperatorsUnderTest []configsections.Operator NameSpaceUnderTest string - Nodes map[string]configsections.Node CrdNames []string + NodesUnderTest map[string]*NodeConfig + // ContainersToExcludeFromConnectivityTests is a set used for storing the containers that should be excluded from // connectivity testing. ContainersToExcludeFromConnectivityTests map[configsections.ContainerIdentifier]interface{} @@ -170,9 +199,20 @@ func (env *TestEnvironment) LoadAndRefresh() { } env.doAutodiscover() } else if env.needsRefresh { + log.Debug("clean up environment Test structure") env.Config.Partner = configsections.TestPartner{} env.Config.TestTarget = configsections.TestTarget{} env.TestOrchestrator = nil + for name, node := range env.NodesUnderTest { + if node.HasDebugPod() { + node.Oc.Close() + autodiscover.DeleteDebugLabel(name) + } + } + env.NodesUnderTest = nil + env.Config.Nodes = nil + env.DebugContainers = nil + log.Debug("start auto discovery") env.doAutodiscover() } } @@ -185,26 +225,116 @@ func (env *TestEnvironment) doAutodiscover() { if autodiscover.PerformAutoDiscovery() { autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget, env.NameSpaceUnderTest) } - autodiscover.FindTestPartner(&env.Config.Partner, env.NameSpaceUnderTest) env.ContainersToExcludeFromConnectivityTests = make(map[configsections.ContainerIdentifier]interface{}) for _, cid := range env.Config.ExcludeContainersFromConnectivityTests { env.ContainersToExcludeFromConnectivityTests[cid] = "" } + env.ContainersUnderTest = env.createContainers(env.Config.ContainerConfigList) env.PodsUnderTest = env.Config.PodsUnderTest + + for _, cid := range env.Config.Partner.ContainersDebugList { + env.ContainersToExcludeFromConnectivityTests[cid.ContainerIdentifier] = "" + } + autodiscover.FindTestPartner(&env.Config.Partner, env.NameSpaceUnderTest) env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest env.OperatorsUnderTest = env.Config.Operators - env.Nodes = env.Config.Nodes - env.CrdNames = autodiscover.FindTestCrdNames(env.Config.CrdFilters) + + env.discoverNodes() log.Infof("Test Configuration: %+v", *env) env.needsRefresh = false } +// labelNodes add label to specific nodes so that node selector in debug daemonset +// can be scheduled +func (env *TestEnvironment) labelNodes() { + var masterNode, workerNode string + // make sure at least one worker and one master has debug set to true + for name, node := range env.NodesUnderTest { + if node.IsMaster() && masterNode == "" { + masterNode = name + } + if node.IsMaster() && node.HasDebugPod() { + masterNode = "" + break + } + } + for name, node := range env.NodesUnderTest { + if node.IsWorker() && workerNode == "" { + workerNode = name + } + if node.IsMaster() && node.HasDebugPod() { + workerNode = "" + break + } + } + if masterNode != "" { + env.NodesUnderTest[masterNode].debug = true + } + if workerNode != "" { + env.NodesUnderTest[workerNode].debug = true + } + // label all nodes + for nodeName, node := range env.NodesUnderTest { + if node.HasDebugPod() { + autodiscover.AddDebugLabel(nodeName) + } + } +} + +// create Nodes data from deployment +func (env *TestEnvironment) createNodes(nodes map[string]configsections.Node) map[string]*NodeConfig { + log.Debug("autodiscovery: create nodes start") + defer log.Debug("autodiscovery: create nodes done") + nodesConfig := make(map[string]*NodeConfig) + for _, n := range nodes { + nodesConfig[n.Name] = &NodeConfig{Node: n, Name: n.Name, Oc: nil, deployment: false} + } + for _, c := range env.ContainersUnderTest { + nodeName := c.ContainerConfiguration.NodeName + if _, ok := nodesConfig[nodeName]; ok { + nodesConfig[nodeName].deployment = true + nodesConfig[nodeName].debug = true + } else { + log.Warn("node ", nodeName, " has deployment, but not the right labels") + } + } + return nodesConfig +} + +// attach debug pod session to node session +func (env *TestEnvironment) AttachDebugPodsToNodes() { + for _, c := range env.DebugContainers { + nodeName := c.ContainerConfiguration.NodeName + if _, ok := env.NodesUnderTest[nodeName]; ok { + env.NodesUnderTest[nodeName].Oc = c.Oc + } + } +} + +// discoverNodes find all the nodes in the cluster +// label the ones with deployment +// attach them to debug pods +func (env *TestEnvironment) discoverNodes() { + env.NodesUnderTest = env.createNodes(env.Config.Nodes) + env.labelNodes() + if !autodiscover.IsMinikube() { + autodiscover.CheckDebugDaemonset() + autodiscover.FindDebugPods(&env.Config.Partner) + for _, debugPod := range env.Config.Partner.ContainersDebugList { + env.ContainersToExcludeFromConnectivityTests[debugPod.ContainerIdentifier] = "" + } + env.DebugContainers = env.createContainers(env.Config.Partner.ContainersDebugList) + } + + env.AttachDebugPodsToNodes() +} + // createContainers contains the general steps involved in creating "oc" sessions and other configuration. A map of the // aggregate information is returned. func (env *TestEnvironment) createContainers(containerDefinitions []configsections.ContainerConfig) map[configsections.ContainerIdentifier]*Container { diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 31e792696..ad9882aa4 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -70,6 +70,8 @@ type TestPartner struct { ContainerConfigList []ContainerConfig `yaml:"partnerContainers" json:"partnerContainers"` // TestOrchestratorID is the id of the partner container for conducting connectivity tests TestOrchestratorID ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` + // DebugPods + ContainersDebugList []ContainerConfig `yaml:"debugContainers,omitempty" json:"debugContainers,omitempty"` } // TestTarget is a collection of resources under test diff --git a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go index bdcce74c2..ccb290c6e 100644 --- a/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go +++ b/pkg/tnf/handlers/bootconfigentries/bootconfigentries.go @@ -21,7 +21,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -39,12 +38,12 @@ type BootConfigEntries struct { } // NewBootConfigEntries creates a BootConfigEntries tnf.Test. -func NewBootConfigEntries(timeout time.Duration, nodeName string) *BootConfigEntries { +func NewBootConfigEntries(timeout time.Duration) *BootConfigEntries { return &BootConfigEntries{ timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"ls /host/boot/loader/entries/\"", "|", common.GetOcDebugCommand(), "--preserve-pod=true", "node/" + nodeName, + "ls /host/boot/loader/entries/", }, } } diff --git a/pkg/tnf/handlers/bootconfigentries/bootconfigentries_test.go b/pkg/tnf/handlers/bootconfigentries/bootconfigentries_test.go index 9335b0818..281b694e7 100644 --- a/pkg/tnf/handlers/bootconfigentries/bootconfigentries_test.go +++ b/pkg/tnf/handlers/bootconfigentries/bootconfigentries_test.go @@ -27,13 +27,13 @@ import ( ) func TestNewBootConfigEntries(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) assert.Equal(t, tnf.ERROR, newBootConfig.Result()) } func Test_ReelFirst(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) firstStep := newBootConfig.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -43,7 +43,7 @@ func Test_ReelFirst(t *testing.T) { } func Test_ReelMatch(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) step := newBootConfig.ReelMatch("", "", testInput) assert.Nil(t, step) @@ -52,7 +52,7 @@ func Test_ReelMatch(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) newBootConfig.ReelEOF() } @@ -62,5 +62,4 @@ const ( testInput = `ostree-1-rhcos.conf ostree-2-rhcos.conf ` - testNodeName = "crc-l6qvn-master-0" ) diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 67b396337..0c2652351 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -20,7 +20,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -96,8 +95,8 @@ func (p *CnfFsDiff) ReelEOF() { } // Command returns command line args for checking the fs difference between a container and it's image -func Command(containerID, nodeName string) []string { - return []string{"echo", "-e", "\"chroot /host\n\"", "podman", "diff", "--format", "json", containerID, "|", common.GetOcDebugCommand(), "node/" + nodeName} +func Command(containerID string) []string { + return []string{"chroot", "/host", "podman", "diff", "--format", "json", containerID} } // NewFsDiff creates a new `FsDiff` test which checks the fs difference between a container and it's image @@ -105,7 +104,7 @@ func NewFsDiff(timeout time.Duration, containerID, nodeName string) *CnfFsDiff { return &CnfFsDiff{ result: tnf.SUCCESS, timeout: timeout, - args: Command(containerID, nodeName), + args: Command(containerID), } } diff --git a/pkg/tnf/handlers/command/command.json b/pkg/tnf/handlers/command/command.json index 6c74aafd0..400468de0 100644 --- a/pkg/tnf/handlers/command/command.json +++ b/pkg/tnf/handlers/command/command.json @@ -6,11 +6,11 @@ }, "description": "Handler to execute user custom commands.", "testResult": 0, - "testTimeout": {{ .TIMEOUT}}, + "testTimeout": {{ .TIMEOUT }}, "reelFirstStep": { - "execute": "{{ .COMMAND}}", + "execute": "{{ .COMMAND }}", "expect":[ "(?m).*"], - "timeout": {{ .TIMEOUT}} + "timeout": {{ .TIMEOUT }} }, "resultContexts":[ { diff --git a/pkg/tnf/handlers/command/command_test.go b/pkg/tnf/handlers/command/command_test.go index a2dc6bfbf..04ac2ee6b 100644 --- a/pkg/tnf/handlers/command/command_test.go +++ b/pkg/tnf/handlers/command/command_test.go @@ -35,10 +35,10 @@ const ( ) var ( + pathRelativeToRoot = path.Join("..", "..", "..", "..") genericTestSchemaFile = path.Join("schemas", "generic-test.schema.json") checkSubFilename = "command.json" expectedPassPattern = "(?m).*" - pathRelativeToRoot = path.Join("..", "..", "..", "..") pathToTestSchemaFile = path.Join(pathRelativeToRoot, genericTestSchemaFile) testCommand = "oc get pods -n tnf" ) diff --git a/pkg/tnf/handlers/common/command.go b/pkg/tnf/handlers/common/command.go index ed84c8896..6c6366090 100644 --- a/pkg/tnf/handlers/common/command.go +++ b/pkg/tnf/handlers/common/command.go @@ -1,16 +1,30 @@ package common +import "strings" + // OcDebugImageID can be set by the test application that uses the test handlers which require the oc debug command to work with a specific image version var OcDebugImageID string const ( - ocDebugCmdBase = "oc debug" + ocCommand = "oc" + debugCommand = "debug" + imageArg = "--image" ) // GetOcDebugCommand returns the command base for any test handler that uses the oc debug command func GetOcDebugCommand() string { + args := []string{ocCommand, debugCommand} + if len(OcDebugImageID) > 0 { + args = append(args, imageArg, OcDebugImageID) + } + return strings.Join(args, " ") +} + +// GetDebugCommand returns the command base for any test handler that uses the debug command +func GetDebugCommand() string { + args := []string{debugCommand} if len(OcDebugImageID) > 0 { - return ocDebugCmdBase + " --image " + OcDebugImageID + args = append(args, imageArg, OcDebugImageID) } - return ocDebugCmdBase + return strings.Join(args, " ") } diff --git a/pkg/tnf/handlers/daemonset/daemonset.go b/pkg/tnf/handlers/daemonset/daemonset.go new file mode 100644 index 000000000..98b52446a --- /dev/null +++ b/pkg/tnf/handlers/daemonset/daemonset.go @@ -0,0 +1,159 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package daemonset + +import ( + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +type Status struct { + Name string + Desired int + Current int + Ready int + Available int + Misscheduled int +} + +// DaemonSet is the reel handler struct. +type DaemonSet struct { + result int + timeout time.Duration + args []string + status Status +} + +const ( + dsRegex = "(?s).+" +) + +// NewDaemonSet returns a new DaemonSet handler struct. +func NewDaemonSet(timeout time.Duration, daemonset, namespace string) *DaemonSet { + return &DaemonSet{ + timeout: timeout, + result: tnf.ERROR, + args: []string{"oc", "-n", namespace, "get", "ds", daemonset, "-o", + "go-template='{{ .spec.template.metadata.name }} ", + "{{ .status.desiredNumberScheduled }}", + "{{ .status.currentNumberScheduled }}", + "{{ .status.numberAvailable }}", + "{{ .status.numberReady }}", + "{{ .status.numberMisscheduled }} {{ printf \"\\n\" }}'", + }, + status: Status{}, + } +} + +// Args returns the initial execution/send command strings for handler DaemonSet. +func (ds *DaemonSet) Args() []string { + return ds.args +} + +// GetIdentifier returns the tnf.Test specific identifier. +func (ds *DaemonSet) GetIdentifier() identifier.Identifier { + // Return the DaemonSet handler identifier. + return identifier.DaemonSetIdentifier +} + +// Timeout returns the timeout for the test. +func (ds *DaemonSet) Timeout() time.Duration { + return ds.timeout +} + +// Result returns the test result. +func (ds *DaemonSet) Result() int { + return ds.result +} + +// ReelFirst returns a reel step for handler DaemonSet. +func (ds *DaemonSet) ReelFirst() *reel.Step { + return &reel.Step{ + Expect: []string{dsRegex}, + Timeout: ds.timeout, + } +} + +// ReelMatch parses the DaemonSet output and set the test result on match. +func (ds *DaemonSet) ReelMatch(_, _, match string) *reel.Step { + const numExpectedFields = 6 + trimmedMatch := strings.Trim(match, "\n") + lines := strings.Split(trimmedMatch, "\n")[0:] // Keep First line only + + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != numExpectedFields { + return nil + } + err := processResult(&ds.status, fields) + if err != nil { + log.Error("Error processing output ", err) + ds.status = Status{} + return nil + } + ds.result = tnf.SUCCESS + return nil + } + return nil +} +func processResult(status *Status, fields []string) error { + var err error + status.Name = fields[0] + status.Desired, err = strconv.Atoi(fields[1]) + if err != nil { + return err + } + status.Current, err = strconv.Atoi(fields[2]) + if err != nil { + return nil + } + status.Available, err = strconv.Atoi(fields[3]) + if err != nil { + return nil + } + status.Ready, err = strconv.Atoi(fields[4]) + if err != nil { + return nil + } + status.Misscheduled, err = strconv.Atoi(fields[5]) + if err != nil { + return nil + } + return nil +} + +// ReelTimeout function for DaemonSet will be called by the reel FSM when a expect timeout occurs. +func (ds *DaemonSet) ReelTimeout() *reel.Step { + return nil +} + +// ReelEOF function for DaemonSet will be called by the reel FSM when a EOF is read. +func (ds *DaemonSet) ReelEOF() { +} + +func (ds *DaemonSet) GetStatus() Status { + return ds.status +} diff --git a/pkg/tnf/handlers/daemonset/daemonset_test.go b/pkg/tnf/handlers/daemonset/daemonset_test.go new file mode 100644 index 000000000..7394ff745 --- /dev/null +++ b/pkg/tnf/handlers/daemonset/daemonset_test.go @@ -0,0 +1,121 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package daemonset + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" +) + +const ( + testTimeoutDuration = 10 * time.Second + testNamespace = "default" + testDebugDaemonset = "debug" + testDataDirectory = "testdata" +) + +type DaemonSetTest struct { + daemonset Status + result int +} + +var testCases = map[string]DaemonSetTest{ + "valid_daemonset": {daemonset: Status{ + Name: "debug", + Desired: 1, + Current: 1, + Ready: 1, + Available: 1, + Misscheduled: 0, + }, result: tnf.SUCCESS}, + "non_valid_daemonset": {daemonset: Status{ + Name: "test", + Desired: 2, + Current: 1, + Ready: 1, + Available: 1, + Misscheduled: 0, + }, result: tnf.SUCCESS}, + "non_valid_output": {daemonset: Status{ + Name: "", + Desired: 0, + Current: 0, + Ready: 0, + Available: 0, + Misscheduled: 0, + }, result: tnf.ERROR}, +} + +func getMockOutputFilename(testName string) string { + return path.Join(testDataDirectory, testName) +} + +func getMockOutput(t *testing.T, testName string) string { + fileName := getMockOutputFilename(testName) + b, err := os.ReadFile(fileName) + assert.Nil(t, err) + return string(b) +} + +// Test_NewDaemonSet is the unit test for NewDaemonSet(). +func Test_NewDaemonSet(t *testing.T) { + newDs := NewDaemonSet(testTimeoutDuration, testDebugDaemonset, testNamespace) + assert.NotNil(t, newDs) + assert.Equal(t, testTimeoutDuration, newDs.Timeout()) + assert.Equal(t, newDs.Result(), tnf.ERROR) + assert.NotNil(t, newDs.GetStatus()) +} + +// Test_DaemonSet_GetIdentifier is the unit test for DaemonSet_GetIdentifier(). +func TestDaemonSet_GetIdentifier(t *testing.T) { + newDs := NewDaemonSet(testTimeoutDuration, testDebugDaemonset, testNamespace) + assert.Equal(t, identifier.DaemonSetIdentifier, newDs.GetIdentifier()) +} + +// Test_DaemonSet_ReelEOF is the unit test for DaemonSet_ReelEOF(). +func TestDaemonSet_ReelEOF(t *testing.T) { + newDs := NewDaemonSet(testTimeoutDuration, testDebugDaemonset, testNamespace) + assert.NotNil(t, newDs) + newDs.ReelEOF() +} + +// Test_DaemonSet_ReelTimeout is the unit test for DaemonSet}_ReelTimeout(). +func TestDaemonSet_ReelTimeout(t *testing.T) { + ds := NewDaemonSet(testTimeoutDuration, "debug", "default") + step := ds.ReelTimeout() + assert.Nil(t, step) +} + +// Test_DaemonSet_ReelMatch is the unit test for DaemonSet_ReelMatch(). +func TestDaemonSet_ReelMatch(t *testing.T) { + for testName, testCase := range testCases { + fmt.Println("process case ", testName) + ds := NewDaemonSet(testTimeoutDuration, testCase.daemonset.Name, "default") + matchMock := getMockOutput(t, testName) + step := ds.ReelMatch("", "", matchMock) + assert.Nil(t, step) + assert.Equal(t, testCase.daemonset, ds.GetStatus()) + assert.Equal(t, testCase.result, ds.result) + } +} diff --git a/pkg/tnf/handlers/daemonset/doc.go b/pkg/tnf/handlers/daemonset/doc.go new file mode 100644 index 000000000..1430c98df --- /dev/null +++ b/pkg/tnf/handlers/daemonset/doc.go @@ -0,0 +1,17 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package daemonset provides a test for daemonset. +package daemonset diff --git a/pkg/tnf/handlers/daemonset/testdata/non_valid_daemonset b/pkg/tnf/handlers/daemonset/testdata/non_valid_daemonset new file mode 100644 index 000000000..7f092794c --- /dev/null +++ b/pkg/tnf/handlers/daemonset/testdata/non_valid_daemonset @@ -0,0 +1,2 @@ +test 2 1 1 1 0 + % \ No newline at end of file diff --git a/pkg/tnf/handlers/daemonset/testdata/non_valid_output b/pkg/tnf/handlers/daemonset/testdata/non_valid_output new file mode 100644 index 000000000..09aa7ea33 --- /dev/null +++ b/pkg/tnf/handlers/daemonset/testdata/non_valid_output @@ -0,0 +1 @@ +debug 2 2 0 0 diff --git a/pkg/tnf/handlers/daemonset/testdata/valid_daemonset b/pkg/tnf/handlers/daemonset/testdata/valid_daemonset new file mode 100644 index 000000000..1da1f8c7d --- /dev/null +++ b/pkg/tnf/handlers/daemonset/testdata/valid_daemonset @@ -0,0 +1,2 @@ +debug 1 1 1 1 0 + % \ No newline at end of file diff --git a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go index cb9a826fc..e8e500ef6 100644 --- a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go +++ b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go @@ -47,7 +47,7 @@ func NewDeploymentsDrain(timeout time.Duration, nodeName string) *DeploymentsDra result: tnf.ERROR, args: []string{ "oc", "adm", "drain", nodeName, "--pod-selector=pod-template-hash", "--disable-eviction=true", - "--delete-local-data=true", "--ignore-daemonsets=true", "--timeout=" + drainTimeoutString, + "--delete-emptydir-data=true", "--ignore-daemonsets=true", "--timeout=" + drainTimeoutString, "&&", "echo", "SUCCESS", }, node: nodeName, diff --git a/pkg/tnf/handlers/nodedebug/nodedebug.go b/pkg/tnf/handlers/nodedebug/nodedebug.go index 0d37e86a1..9ea8fc8c7 100644 --- a/pkg/tnf/handlers/nodedebug/nodedebug.go +++ b/pkg/tnf/handlers/nodedebug/nodedebug.go @@ -21,7 +21,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -51,7 +50,7 @@ func NewNodeDebug(timeout time.Duration, nodeName, command string, trim, split b timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "-e", "\"chroot /host\n\"", command, "|", common.GetOcDebugCommand(), "node/" + nodeName, + "chroot", "/host", command, }, Trim: trim, Split: split, diff --git a/pkg/tnf/handlers/nodehugepages/nodehugepages.go b/pkg/tnf/handlers/nodehugepages/nodehugepages.go index 3375a4b80..685963915 100644 --- a/pkg/tnf/handlers/nodehugepages/nodehugepages.go +++ b/pkg/tnf/handlers/nodehugepages/nodehugepages.go @@ -22,7 +22,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -41,14 +40,14 @@ type NodeHugepages struct { } // NewNodeHugepages creates a new NodeHugepages tnf.Test. -func NewNodeHugepages(timeout time.Duration, node string, hugepagesz, hugepages int) *NodeHugepages { +func NewNodeHugepages(timeout time.Duration, hugepagesz, hugepages int) *NodeHugepages { return &NodeHugepages{ hugepagesz: hugepagesz, hugepages: hugepages, timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"grep -E 'HugePages_Total:|Hugepagesize:' /proc/meminfo\"", "|", common.GetOcDebugCommand(), "node/" + node, + "grep -E 'HugePages_Total:|Hugepagesize:' /proc/meminfo", }, } } diff --git a/pkg/tnf/handlers/nodehugepages/nodehugepages_test.go b/pkg/tnf/handlers/nodehugepages/nodehugepages_test.go index 54b9df075..260c5284b 100644 --- a/pkg/tnf/handlers/nodehugepages/nodehugepages_test.go +++ b/pkg/tnf/handlers/nodehugepages/nodehugepages_test.go @@ -27,14 +27,14 @@ import ( ) func Test_NewNodeHugepages(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) assert.Equal(t, testTimeoutDuration, newNh.Timeout()) assert.Equal(t, newNh.Result(), tnf.ERROR) } func Test_ReelFirstPositive(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) firstStep := newNh.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -44,7 +44,7 @@ func Test_ReelFirstPositive(t *testing.T) { } func Test_ReelFirstNegative(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) firstStep := newNh.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -53,7 +53,7 @@ func Test_ReelFirstNegative(t *testing.T) { } func Test_ReelMatchSuccess(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) step := newNh.ReelMatch("", "", testInputSuccess) assert.Nil(t, step) @@ -61,7 +61,7 @@ func Test_ReelMatchSuccess(t *testing.T) { } func Test_ReelMatchFailure(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) step := newNh.ReelMatch("", "", testInputFailure) assert.Nil(t, step) @@ -70,14 +70,13 @@ func Test_ReelMatchFailure(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newNh := nh.NewNodeHugepages(testTimeoutDuration, testNode, testExpectedHugepagesz, testExpectedHugepages) + newNh := nh.NewNodeHugepages(testTimeoutDuration, testExpectedHugepagesz, testExpectedHugepages) assert.NotNil(t, newNh) newNh.ReelEOF() } const ( testTimeoutDuration = time.Second * 2 - testNode = "testNode" testInputError = "" testInputSuccess = "HugePages_Total: 64\nHugepagesize: 1048576 kB\n" testInputFailure = "HugePages_Total: 32\nHugepagesize: 1000000 kB\n" diff --git a/pkg/tnf/handlers/nodetainted/nodetainted.go b/pkg/tnf/handlers/nodetainted/nodetainted.go index b11bd57c6..320e17897 100644 --- a/pkg/tnf/handlers/nodetainted/nodetainted.go +++ b/pkg/tnf/handlers/nodetainted/nodetainted.go @@ -20,7 +20,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -38,12 +37,12 @@ type NodeTainted struct { } // NewNodeTainted creates a new NodeTainted tnf.Test. -func NewNodeTainted(timeout time.Duration, nodeName string) *NodeTainted { +func NewNodeTainted(timeout time.Duration) *NodeTainted { return &NodeTainted{ timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "cat", "/proc/sys/kernel/tainted", "|", common.GetOcDebugCommand(), "node/" + nodeName, + "cat", "/proc/sys/kernel/tainted", }, } } diff --git a/pkg/tnf/handlers/nodetainted/nodetainted_test.go b/pkg/tnf/handlers/nodetainted/nodetainted_test.go index dfe7a13f6..70eae8541 100644 --- a/pkg/tnf/handlers/nodetainted/nodetainted_test.go +++ b/pkg/tnf/handlers/nodetainted/nodetainted_test.go @@ -27,14 +27,14 @@ import ( ) func Test_NewNodeTainted(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) assert.Equal(t, testTimeoutDuration, newNt.Timeout()) assert.Equal(t, newNt.Result(), tnf.ERROR) } func Test_ReelFirstPositiveSuccess(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) firstStep := newNt.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -44,7 +44,7 @@ func Test_ReelFirstPositiveSuccess(t *testing.T) { } func Test_ReelFirstPositiveFailure(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) firstStep := newNt.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -54,7 +54,7 @@ func Test_ReelFirstPositiveFailure(t *testing.T) { } func Test_ReelFirstNegative(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) firstStep := newNt.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -63,7 +63,7 @@ func Test_ReelFirstNegative(t *testing.T) { } func Test_ReelMatchSuccess(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) step := newNt.ReelMatch("", "", testMatchSuccess) assert.Nil(t, step) @@ -71,7 +71,7 @@ func Test_ReelMatchSuccess(t *testing.T) { } func Test_ReelMatchFail(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) step := newNt.ReelMatch("", "", testMatchFailure) assert.Nil(t, step) @@ -80,13 +80,12 @@ func Test_ReelMatchFail(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newNt := nt.NewNodeTainted(testTimeoutDuration, testNodeName) + newNt := nt.NewNodeTainted(testTimeoutDuration) assert.NotNil(t, newNt) newNt.ReelEOF() } const ( - testNodeName = "testNode" testTimeoutDuration = time.Second * 2 testInputError = "" testInputFailure = "1\n" diff --git a/pkg/tnf/handlers/readbootconfig/readbootconfig.go b/pkg/tnf/handlers/readbootconfig/readbootconfig.go index aae83fa75..2463426e4 100644 --- a/pkg/tnf/handlers/readbootconfig/readbootconfig.go +++ b/pkg/tnf/handlers/readbootconfig/readbootconfig.go @@ -20,7 +20,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -38,13 +37,12 @@ type ReadBootConfig struct { } // NewReadBootConfig creates a ReadBootConfig tnf.Test. -func NewReadBootConfig(timeout time.Duration, nodeName /*, entryName*/ string) *ReadBootConfig { +func NewReadBootConfig(timeout time.Duration) *ReadBootConfig { return &ReadBootConfig{ timeout: timeout, result: tnf.ERROR, args: []string{ - // "echo", "\"cat /host/boot/loader/entries/" + entryName + "\"", "|", common.GetOcDebugCommand(), "--preserve-pod=true", "node/" + nodeName, - "echo", "\"cat /host/boot/loader/entries/\\`ls /host/boot/loader/entries/ | sort | tail -n 1\\`\"", "|", common.GetOcDebugCommand(), "-q", "node/" + nodeName, + "cat /host/boot/loader/entries/$(ls /host/boot/loader/entries/ | sort | tail -n 1)", }, } } diff --git a/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go b/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go index f15163363..86bb1a132 100644 --- a/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go +++ b/pkg/tnf/handlers/readbootconfig/readbootconfig_test.go @@ -28,13 +28,13 @@ import ( ) func TestReadBootConfig(t *testing.T) { - newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName) + newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration) assert.NotNil(t, newReadBootConfig) assert.Equal(t, tnf.ERROR, newReadBootConfig.Result()) } func Test_ReelFirst(t *testing.T) { - newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration, testNodeName) + newReadBootConfig := readbootconfig.NewReadBootConfig(testTimeoutDuration) assert.NotNil(t, newReadBootConfig) firstStep := newReadBootConfig.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -44,7 +44,7 @@ func Test_ReelFirst(t *testing.T) { } func Test_ReelMatch(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) step := newBootConfig.ReelMatch("", "", testInput) assert.Nil(t, step) @@ -53,7 +53,7 @@ func Test_ReelMatch(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration, testNodeName) + newBootConfig := bootconfigentries.NewBootConfigEntries(testTimeoutDuration) assert.NotNil(t, newBootConfig) newBootConfig.ReelEOF() } @@ -65,6 +65,4 @@ const ( options random.trust_cpu=on console=tty0 linux /ostree/rhcos-8db99645874 initrd /ostree/rhcos-8db99645874` - testNodeName = "crc-l6qvn-master-0" - // testBootEntryName = "ostree-2-rhcos.conf" ) diff --git a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go index 419df701c..d892072f8 100644 --- a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go +++ b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs.go @@ -20,7 +20,6 @@ import ( "time" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) @@ -39,12 +38,12 @@ type SysctlAllConfigsArgs struct { } // NewSysctlAllConfigsArgs creates a SysctlAllConfigsArgs tnf.Test. -func NewSysctlAllConfigsArgs(timeout time.Duration, nodeName string) *SysctlAllConfigsArgs { +func NewSysctlAllConfigsArgs(timeout time.Duration) *SysctlAllConfigsArgs { return &SysctlAllConfigsArgs{ timeout: timeout, result: tnf.ERROR, args: []string{ - "echo", "\"sysctl --system\"", "|", common.GetOcDebugCommand(), "-q", "node/" + nodeName, + "sysctl --system", }, } } diff --git a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs_test.go b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs_test.go index 6b774b96d..9651f478a 100644 --- a/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs_test.go +++ b/pkg/tnf/handlers/sysctlallconfigsargs/sysctlallconfigsargs_test.go @@ -27,13 +27,13 @@ import ( ) func TestNewSysctlAllConfigsArgs(t *testing.T) { - newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration, testNodeName) + newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration) assert.NotNil(t, newSysctlAllConfigsArgs) assert.Equal(t, tnf.ERROR, newSysctlAllConfigsArgs.Result()) } func Test_ReelFirst(t *testing.T) { - newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration, testNodeName) + newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration) assert.NotNil(t, newSysctlAllConfigsArgs) firstStep := newSysctlAllConfigsArgs.ReelFirst() re := regexp.MustCompile(firstStep.Expect[0]) @@ -43,7 +43,7 @@ func Test_ReelFirst(t *testing.T) { } func Test_ReelMatch(t *testing.T) { - newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration, testNodeName) + newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration) assert.NotNil(t, newSysctlAllConfigsArgs) step := newSysctlAllConfigsArgs.ReelMatch("", "", testInput) assert.Nil(t, step) @@ -52,7 +52,7 @@ func Test_ReelMatch(t *testing.T) { // Just ensure there are no panics. func Test_ReelEof(t *testing.T) { - newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration, testNodeName) + newSysctlAllConfigsArgs := sysctlallconfigsargs.NewSysctlAllConfigsArgs(testTimeoutDuration) assert.NotNil(t, newSysctlAllConfigsArgs) newSysctlAllConfigsArgs.ReelEOF() } @@ -88,5 +88,4 @@ const ( fs.inotify.max_user_instances = 8192 * Applying /etc/sysctl.conf ... ` - testNodeName = "crc-l6qvn-master-0" ) diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index d7e47cb97..e051908ef 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -61,6 +61,7 @@ const ( csiDriverIdentifierURL = "http://test-network-function.com/tests/csiDriver" clusterVersionIdentifierURL = "http://test-network-function.com/tests/clusterVersion" crdStatusExistenceIdentifierURL = "http://test-network-function.com/tests/crdStatusExistence" + daemonSetIdentifierURL = "http://test-network-function.com/tests/daemonset" versionOne = "v1.0.0" ) @@ -635,6 +636,18 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.JqBinaryName, }, }, + daemonSetIdentifierURL: { + Identifier: DaemonSetIdentifier, + Description: "check whether a given daemonset was deployed successfully", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + }, + }, } // CommandIdentifier is the Identifier used to represent the generic command test case. @@ -889,3 +902,8 @@ var CrdStatusExistenceIdentifier = Identifier{ URL: crdStatusExistenceIdentifierURL, SemanticVersion: versionOne, } + +var DaemonSetIdentifier = Identifier{ + URL: daemonSetIdentifierURL, + SemanticVersion: versionOne, +} diff --git a/pkg/tnf/interactive/oc.go b/pkg/tnf/interactive/oc.go index 183b0690b..9b83e6e62 100644 --- a/pkg/tnf/interactive/oc.go +++ b/pkg/tnf/interactive/oc.go @@ -20,22 +20,21 @@ import ( "time" expect "github.com/google/goexpect" + log "github.com/sirupsen/logrus" ) const ( - ocClientCommandSeparator = "--" - ocCommand = "oc" - ocContainerArg = "-c" - ocDefaultShell = "sh" - ocExecCommand = "exec" - ocNamespaceArg = "-n" - ocInteractiveArg = "-it" + ocCommand = "oc" + ocContainerArg = "-c" + ocRsh = "rsh" + ocNamespaceArg = "-n" ) // Oc provides an OpenShift Client designed to wrap the "oc" CLI. type Oc struct { // name of the pod pod string + // node set to true means the sessions is node session // name of the container container string // namespace of the pod @@ -44,7 +43,7 @@ type Oc struct { serviceAccountName string // timeout for commands run in expecter timeout time.Duration - // options for experter, such as expect.Verbose(true) + // options for expecter, such as expect.Verbose(true) opts []Option // the underlying subprocess implementation, tailored to OpenShift Client expecter *expect.Expecter @@ -58,7 +57,7 @@ type Oc struct { // SpawnOc creates an OpenShift Client subprocess, spawning the appropriate underlying PTY. func SpawnOc(spawner *Spawner, pod, container, namespace string, timeout time.Duration, opts ...Option) (*Oc, <-chan error, error) { - ocArgs := []string{ocExecCommand, ocNamespaceArg, namespace, ocInteractiveArg, pod, ocContainerArg, container, ocClientCommandSeparator, ocDefaultShell} + ocArgs := []string{ocRsh, ocNamespaceArg, namespace, ocContainerArg, container, pod} context, err := (*spawner).Spawn(ocCommand, ocArgs, timeout, opts...) if err != nil { return nil, context.GetErrorChannel(), err @@ -114,10 +113,16 @@ func (o *Oc) GetErrorChannel() <-chan error { // GetDoneChannel returns the receive only done channel func (o *Oc) GetDoneChannel() <-chan bool { + log.Debugf("read done channel pod %s/%s", o.pod, o.container) return o.doneChannel } // Close sends the signal to the done channel func (o *Oc) Close() { + if o == nil { + log.Debugf("Oc is null, nothing to close") + return + } + log.Debugf("send close to channel pod %s/%s ", o.pod, o.container) o.doneChannel <- true } diff --git a/pkg/tnf/interactive/shell.go b/pkg/tnf/interactive/shell.go index b4cb6b49f..fe0afb18d 100644 --- a/pkg/tnf/interactive/shell.go +++ b/pkg/tnf/interactive/shell.go @@ -17,12 +17,15 @@ package interactive import ( + "log" "os" "time" ) const ( shellEnvironmentVariableKey = "SHELL" + defaultTimeoutSeconds = 10 + defaultTimeout = defaultTimeoutSeconds * time.Second ) // SpawnShell creates an interactive shell subprocess based on the value of $SHELL, spawning the appropriate underlying @@ -32,3 +35,14 @@ func SpawnShell(spawner *Spawner, timeout time.Duration, opts ...Option) (*Conte var args []string return (*spawner).Spawn(shellEnv, args, timeout, opts...) } + +// +// +// GetContext spawns a new shell session and returns its context +func GetContext() *Context { + context, err := SpawnShell(CreateGoExpectSpawner(), defaultTimeout, Verbose(true), SendTimeout(defaultTimeout)) + if err != nil || context == nil || context.GetExpecter() == nil { + log.Panicf("can't get a proper context for test execution") + } + return context +} diff --git a/test-network-function/common/constant.go b/test-network-function/common/constant.go index a6183b6d3..3a124a334 100644 --- a/test-network-function/common/constant.go +++ b/test-network-function/common/constant.go @@ -28,4 +28,5 @@ const ( ObservabilityTestKey = "observability" OperatorTestKey = "operator" PlatformAlterationTestKey = "platform-alteration" + CommonTestKey = "common" ) diff --git a/test-network-function/common/suite.go b/test-network-function/common/suite.go new file mode 100644 index 000000000..ea6269491 --- /dev/null +++ b/test-network-function/common/suite.go @@ -0,0 +1,45 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package common + +import ( + "github.com/onsi/ginkgo" + log "github.com/sirupsen/logrus" + configpkg "github.com/test-network-function/test-network-function/pkg/config" + "github.com/test-network-function/test-network-function/pkg/config/autodiscover" +) + +var _ = ginkgo.Describe(CommonTestKey, func() { + var env *configpkg.TestEnvironment + + ginkgo.BeforeSuite(func() { + for name := range autodiscover.GetNodesList() { + autodiscover.DeleteDebugLabel(name) + } + env = configpkg.GetTestEnvironment() + env.LoadAndRefresh() + }) + + ginkgo.AfterSuite(func() { + // clean up added label to nodes + log.Info("clean up added labels to nodes") + for name, node := range env.NodesUnderTest { + node.Oc.Close() + autodiscover.DeleteDebugLabel(name) + } + }) +}) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index eeec737df..f2a35fecc 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -74,7 +74,12 @@ var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) ginkgo.When("a cluster is set up and installed with OpenShift", func() { - + env := config.GetTestEnvironment() + ginkgo.BeforeEach(func() { + env.LoadAndRefresh() + gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) + gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) + }) ginkgo.By("should report OCP version") testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestclusterVersionIdentifier) ginkgo.It(testID, func() { @@ -177,8 +182,8 @@ func GetCsiDriverInfo() map[string]interface{} { } func getMasterNodeName(env *config.TestEnvironment) string { - for _, node := range env.Nodes { - if node.IsMaster() { + for _, node := range env.NodesUnderTest { + if node.IsMaster() && node.HasDebugPod() { return node.Name } } @@ -186,8 +191,8 @@ func getMasterNodeName(env *config.TestEnvironment) string { } func getWorkerNodeName(env *config.TestEnvironment) string { - for _, node := range env.Nodes { - if node.IsWorker() { + for _, node := range env.NodesUnderTest { + if node.IsWorker() && node.HasDebugPod() { return node.Name } } @@ -195,9 +200,10 @@ func getWorkerNodeName(env *config.TestEnvironment) string { } func listNodeCniPlugins(nodeName string) []CniPlugin { - const command = "jq -r .name,.cniVersion '/etc/cni/net.d/*'" + const command = "jq -r .name,.cniVersion /etc/cni/net.d/*" result := []CniPlugin{} - context := common.GetContext() + nodes := config.GetTestEnvironment().NodesUnderTest + context := nodes[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -259,7 +265,8 @@ func getNodeLscpu(nodeName string) map[string]string { const command = "lscpu" const numSplitSubstrings = 2 result := map[string]string{} - context := common.GetContext() + env := config.GetTestEnvironment().NodesUnderTest + context := env[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -275,7 +282,8 @@ func getNodeIfconfig(nodeName string) map[string][]string { const command = "ifconfig" const numSplitSubstrings = 2 result := map[string][]string{} - context := common.GetContext() + env := config.GetTestEnvironment().NodesUnderTest + context := env[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -297,7 +305,8 @@ func getNodeIfconfig(nodeName string) map[string][]string { func getNodeLsblk(nodeName string) interface{} { const command = "lsblk -J" - context := common.GetContext() + env := config.GetTestEnvironment().NodesUnderTest + context := env[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, false, false) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -310,7 +319,8 @@ func getNodeLsblk(nodeName string) interface{} { func getNodeLspci(nodeName string) []string { const command = "lspci" - context := common.GetContext() + env := config.GetTestEnvironment().NodesUnderTest + context := env[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test-network-function/lifecycle/doc.go b/test-network-function/lifecycle/doc.go index 76b93ac91..fd41abc52 100644 --- a/test-network-function/lifecycle/doc.go +++ b/test-network-function/lifecycle/doc.go @@ -16,6 +16,6 @@ /* Package lifecycle contains k8s resource lifecycle related tests, such as pod -scheduling, scaling, temination etc. +scheduling, scaling, termination etc. */ package lifecycle diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index b28a48ce1..093a426a6 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -19,7 +19,6 @@ package lifecycle import ( "fmt" "path" - "sort" "strings" "time" @@ -40,7 +39,6 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf" dp "github.com/test-network-function/test-network-function/pkg/tnf/handlers/deployments" dd "github.com/test-network-function/test-network-function/pkg/tnf/handlers/deploymentsdrain" - dn "github.com/test-network-function/test-network-function/pkg/tnf/handlers/deploymentsnodes" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/graceperiod" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodeselector" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/owners" @@ -143,6 +141,8 @@ func restoreDeployments(env *config.TestEnvironment) { } func closeOcSessionsByDeployment(containers map[configsections.ContainerIdentifier]*config.Container, deployment configsections.Deployment) { + log.Debug("close session for deployment=", deployment.Name, " start") + defer log.Debug("close session for deployment=", deployment.Name, " done") for cid, c := range containers { if cid.Namespace == deployment.Namespace && strings.HasPrefix(cid.PodName, deployment.Name+"-") { log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) @@ -168,6 +168,7 @@ func runScalingTest(deployment configsections.Deployment) { test, err := tnf.NewTest(common.GetContext().GetExpecter(), handler, []reel.Handler{handler}, common.GetContext().GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunAndValidate() + // Wait until the deployment is ready waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } @@ -178,17 +179,16 @@ func testScaling(env *config.TestEnvironment) { ginkgo.By("Testing deployment scaling") defer results.RecordResult(identifiers.TestScalingIdentifier) defer restoreDeployments(env) + defer env.SetNeedsRefresh() if len(env.DeploymentsUnderTest) == 0 { ginkgo.Skip("No test deployments found.") } - for _, deployment := range env.DeploymentsUnderTest { ginkgo.By(fmt.Sprintf("Scaling Deployment=%s, Replicas=%d (ns=%s)", deployment.Name, deployment.Replicas, deployment.Namespace)) closeOcSessionsByDeployment(env.ContainersUnderTest, deployment) - replicaCount := deployment.Replicas // ScaleIn, removing one pod from the replicaCount @@ -198,9 +198,6 @@ func testScaling(env *config.TestEnvironment) { // Scaleout, restoring the original replicaCount number deployment.Replicas = replicaCount runScalingTest(deployment) - - // Ensure next tests/test suites receive a refreshed config. - env.SetNeedsRefresh() } }) } @@ -293,8 +290,8 @@ func testPodsRecreation(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRecreationIdentifier) ginkgo.It(testID, func() { - env.SetNeedsRefresh() ginkgo.By("Testing node draining effect of deployment") + defer results.RecordResult(identifiers.TestPodRecreationIdentifier) ginkgo.By(fmt.Sprintf("test deployment in namespace %s", env.NameSpaceUnderTest)) deployments, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) @@ -305,54 +302,29 @@ func testPodsRecreation(env *config.TestEnvironment) { if len(notReadyDeployments) != 0 { ginkgo.Skip("Can not test when deployments are not ready") } - - ginkgo.By("Should return map of nodes to deployments") - nodesSorted := getDeploymentsNodes(env.NameSpaceUnderTest) - + defer env.SetNeedsRefresh() ginkgo.By("should create new replicas when node is drained") - defer results.RecordResult(identifiers.TestPodRecreationIdentifier) - for _, n := range nodesSorted { - closeOcSessionsByNode(env.ContainersUnderTest, n.name) - closeOcSessionsByNode(env.PartnerContainers, n.name) + for _, n := range env.NodesUnderTest { + if !n.HasDeployment() { + log.Debug("node ", n.Name, " has no deployment, skip draining") + continue + } + closeOcSessionsByNode(env.ContainersUnderTest, n.Name) + closeOcSessionsByNode(env.PartnerContainers, n.Name) // drain node - drainNode(n.name) // should go in this + drainNode(n.Name) // should go in this // verify deployments are ready again _, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) if len(notReadyDeployments) != 0 { - uncordonNode(n.name) - ginkgo.Fail(fmt.Sprintf("did not create replicas when node %s is drained", n.name)) + uncordonNode(n.Name) + ginkgo.Fail(fmt.Sprintf("did not create replicas when node %s is drained", n.Name)) } - uncordonNode(n.name) + uncordonNode(n.Name) } }) } -type node struct { - name string - deployments map[string]bool -} - -func sortNodesMap(nodesMap dn.NodesMap) []node { - nodes := make([]node, 0, len(nodesMap)) - for n, d := range nodesMap { - nodes = append(nodes, node{n, d}) - } - sort.Slice(nodes, func(i, j int) bool { return len(nodes[i].deployments) > len(nodes[j].deployments) }) - return nodes -} - -func getDeploymentsNodes(namespace string) []node { - context := common.GetContext() - tester := dn.NewDeploymentsNodes(common.DefaultTimeout, namespace) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidate() - nodes := tester.GetNodes() - gomega.Expect(nodes).NotTo(gomega.BeEmpty()) - return sortNodesMap(nodes) -} - // getDeployments returns map of deployments and names of not-ready deployments func getDeployments(namespace string) (deployments dp.DeploymentMap, notReadyDeployments []string) { context := common.GetContext() @@ -395,6 +367,7 @@ func uncordonNode(node string) { test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) + test.RunAndValidate() } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index ebec4e7d8..4c85370fb 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -132,10 +132,11 @@ func testContainersFsDiff(env *config.TestEnvironment) { for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() containerName := cut.Oc.GetPodContainerName() - context := cut.Oc + containerOC := cut.Oc nodeName := cut.ContainerConfiguration.NodeName ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) - test := newContainerFsDiffTest(nodeName, context) + nodeOc := env.NodesUnderTest[nodeName].Oc + test := newContainerFsDiffTest(nodeName, nodeOc, containerOC) test.RunWithFailureCallback(func() { badContainers = append(badContainers, containerName) ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) @@ -147,7 +148,7 @@ func testContainersFsDiff(env *config.TestEnvironment) { } // newContainerFsDiffTest test that the CUT didn't install new packages after starting, and report through Ginkgo. -func newContainerFsDiffTest(nodeName string, targetContainerOC *interactive.Oc) *tnf.Test { +func newContainerFsDiffTest(nodeName string, nodeOc, targetContainerOC *interactive.Oc) *tnf.Test { defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) targetContainerOC.GetExpecter() containerIDTester := containerid.NewContainerID(common.DefaultTimeout) @@ -155,13 +156,11 @@ func newContainerFsDiffTest(nodeName string, targetContainerOC *interactive.Oc) gomega.Expect(err).To(gomega.BeNil()) test.RunAndValidate() containerID := containerIDTester.GetID() - context := common.GetContext() fsDiffTester := cnffsdiff.NewFsDiff(common.DefaultTimeout, containerID, nodeName) - test, err = tnf.NewTest(context.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, context.GetErrorChannel()) + test, err = tnf.NewTest(nodeOc.GetExpecter(), fsDiffTester, []reel.Handler{fsDiffTester}, nodeOc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) return test } - func getMcKernelArguments(context *interactive.Context, mcName string) map[string]string { mcKernelArgumentsTester := mckernelarguments.NewMcKernelArguments(common.DefaultTimeout, mcName) test, err := tnf.NewTest(context.GetExpecter(), mcKernelArgumentsTester, []reel.Handler{mcKernelArgumentsTester}, context.GetErrorChannel()) @@ -201,8 +200,8 @@ func getCurrentKernelCmdlineArgs(targetContainerOc *interactive.Oc) map[string]s return utils.ArgListToMap(currentSplitKernelCmdlineArgs) } -func getGrubKernelArgs(context *interactive.Context, nodeName string) map[string]string { - readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout, nodeName) +func getGrubKernelArgs(context *interactive.Oc) map[string]string { + readBootConfigTester := readbootconfig.NewReadBootConfig(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), readBootConfigTester, []reel.Handler{readBootConfigTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunAndValidate() @@ -244,8 +243,8 @@ func parseSysctlSystemOutput(sysctlSystemOutput string) map[string]string { return retval } -func getSysctlConfigArgs(context *interactive.Context, nodeName string) map[string]string { - sysctlAllConfigsArgsTester := sysctlallconfigsargs.NewSysctlAllConfigsArgs(common.DefaultTimeout, nodeName) +func getSysctlConfigArgs(context *interactive.Oc) map[string]string { + sysctlAllConfigsArgsTester := sysctlallconfigsargs.NewSysctlAllConfigsArgs(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), sysctlAllConfigsArgsTester, []reel.Handler{sysctlAllConfigsArgsTester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunAndValidate() @@ -273,7 +272,9 @@ func testBootParamsHelper(context *interactive.Context, podName, podNamespace st mcName := getMcName(context, nodeName) mcKernelArgumentsMap := getMcKernelArguments(context, mcName) currentKernelArgsMap := getCurrentKernelCmdlineArgs(targetContainerOc) - grubKernelConfigMap := getGrubKernelArgs(context, nodeName) + env := config.GetTestEnvironment() + nodeOC := env.NodesUnderTest[nodeName].Oc + grubKernelConfigMap := getGrubKernelArgs(nodeOC) for key, mcVal := range mcKernelArgumentsMap { if currentVal, ok := currentKernelArgsMap[key]; ok { @@ -298,7 +299,9 @@ func testSysctlConfigsHelper(podName, podNamespace string) { ginkgo.By(fmt.Sprintf("Testing sysctl config files for the pod's node %s/%s", podNamespace, podName)) context := common.GetContext() nodeName := getPodNodeName(context, podName, podNamespace) - combinedSysctlSettings := getSysctlConfigArgs(context, nodeName) + env := config.GetTestEnvironment() + nodeOc := env.NodesUnderTest[nodeName].Oc + combinedSysctlSettings := getSysctlConfigArgs(nodeOc) mcName := getMcName(context, nodeName) mcKernelArgumentsMap := getMcKernelArguments(context, mcName) for key, sysctlConfigVal := range combinedSysctlSettings { @@ -326,9 +329,9 @@ func testTainted(env *config.TestEnvironment) { ginkgo.By("Testing tainted nodes in cluster") var taintedNodes []string - for _, node := range env.Nodes { - context := common.GetContext() - tester := nodetainted.NewNodeTainted(common.DefaultTimeout, node.Name) + for _, node := range env.NodesUnderTest { + context := node.Oc + tester := nodetainted.NewNodeTainted(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunWithFailureCallback(func() { @@ -370,18 +373,18 @@ func testHugepages(env *config.TestEnvironment) { defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) var badNodes []string - context := common.GetContext() - for _, node := range env.Nodes { + for _, node := range env.NodesUnderTest { if !node.IsWorker() { continue } + nodeOc := node.Oc ginkgo.By("Should return machineconfig hugepages configuration of node " + node.Name) nodeHugePagesCount, nodeHugePagesSize := getNodeMcHugepages(node.Name) ginkgo.By(fmt.Sprintf("Node's machine config hugepages=%d/hugepagesz=%d values should match the actual ones in the node.", nodeHugePagesCount, nodeHugePagesSize)) - tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, node.Name, nodeHugePagesSize, nodeHugePagesCount) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, nodeHugePagesSize, nodeHugePagesCount) + test, err := tnf.NewTest(nodeOc.GetExpecter(), tester, []reel.Handler{tester}, nodeOc.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunWithFailureCallback(func() { badNodes = append(badNodes, node.Name) From fac12280053b79f61c8d3876e55623bd3ceaeecf Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 26 Oct 2021 16:23:59 -0400 Subject: [PATCH 107/344] Increase time out for deploying debug pods (#411) --- pkg/config/autodiscover/autodiscover_debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go index e09127f05..3d6897230 100644 --- a/pkg/config/autodiscover/autodiscover_debug.go +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -84,7 +84,7 @@ func CheckDebugDaemonset() { gomega.Eventually(func() bool { log.Debug("check debug daemonset status") return checkDebugPodsReadiness() - }, DefaultTimeout, 2*time.Second).Should(gomega.Equal(true)) //nolint: gomnd + }, 60*time.Second, 2*time.Second).Should(gomega.Equal(true)) //nolint: gomnd } // checkDebugPodsReadiness helper function that returns true if the daemonset debug is deployed properly From 9bacf5a2b82e03329b0a906f470348f4f7f8521f Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Tue, 26 Oct 2021 23:08:56 -0500 Subject: [PATCH 108/344] Fix for "latest" image tag not created (#412) --- .github/workflows/tnf-image.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 1ea7aaf46..7b1fa4209 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -82,7 +82,6 @@ jobs: - name: Update env variables run: | echo "TNF_VERSION=${{ steps.set_tnf_version.outputs.version_number }}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{ steps.set_tnf_version.outputs.version_number }}" >> $GITHUB_ENV echo "PARTNER_VERSION=${{ steps.set_partner_version.outputs.partner_version_number }}" >> $GITHUB_ENV - name: Ensure $TNF_VERSION and $IMAGE_TAG are set From 835e2c00d13099eb23e0368c165ff4d2af8071bc Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:04:28 -0500 Subject: [PATCH 109/344] Support for more accurate logs in claims and adaptation to Ginkgo V2.0 beta (#397) * Support for more accurate logs in claims and adaptation to Ginkgo V2.0 beta * Update pkg/tnf/handlers/generic/assertion/assertion.go Co-authored-by: Brandon Palm * Update pkg/tnf/identifier/identifier.go Co-authored-by: Brandon Palm * PR review * PR review * lint * PR review * PR Review Salah * bumping test-network-function-claim to 1.0.5 * PR review Salah * rest of merge Co-authored-by: Brandon Palm Co-authored-by: Jun Chen --- go.mod | 5 +- go.sum | 15 ++ pkg/junit/convert.go | 51 +++-- pkg/junit/convert_test.go | 4 +- pkg/junit/testdata/success.junit.xml | 181 ++++++++++++++++-- .../handlers/generic/assertion/assertion.go | 8 +- pkg/tnf/identifier/identifier.go | 8 +- run-cnf-suites.sh | 8 +- test-network-function/accesscontrol/suite.go | 13 +- test-network-function/certification/suite.go | 8 +- test-network-function/common/suite.go | 32 ++-- test-network-function/diagnostic/suite.go | 106 +++++----- test-network-function/generic/suite.go | 9 +- .../identifiers/identifiers.go | 19 +- test-network-function/lifecycle/suite.go | 18 +- test-network-function/networking/suite.go | 12 +- test-network-function/observability/suite.go | 6 +- test-network-function/operator/suite.go | 26 +-- test-network-function/platform/suite.go | 15 +- test-network-function/results/results.go | 58 +++--- test-network-function/suite_test.go | 16 +- 21 files changed, 380 insertions(+), 238 deletions(-) diff --git a/go.mod b/go.mod index 117407cb8..1eaec4feb 100644 --- a/go.mod +++ b/go.mod @@ -10,13 +10,14 @@ require ( github.com/golang/mock v1.6.0 github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f + github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect github.com/kr/pretty v0.2.1 // indirect - github.com/onsi/ginkgo v1.16.5 + github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057 github.com/onsi/gomega v1.16.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 - github.com/test-network-function/test-network-function-claim v1.0.4 + github.com/test-network-function/test-network-function-claim v1.0.5 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect google.golang.org/grpc v1.41.0 diff --git a/go.sum b/go.sum index b49ac7cd0..e8ea75951 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/edcdavid/test-network-function-claim v1.0.6 h1:jb5//83SVvvxII2BJlJJpc3YDHv0L16AoZqGiCrgugI= +github.com/edcdavid/test-network-function-claim v1.0.6/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= +github.com/edcdavid/test-network-function-claim v1.0.7 h1:77GzxaTzyLTL2eqNb4DXnjSXS068+mo4QJg5yZRy1Lk= +github.com/edcdavid/test-network-function-claim v1.0.7/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= +github.com/edcdavid/test-network-function-claim v1.0.8 h1:WgZkLc+jK1IlMFr2w3zxp42+0sRskDMxlONCGfS2ZNg= +github.com/edcdavid/test-network-function-claim v1.0.8/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -154,6 +160,9 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 h1:zHs+jv3LO743/zFGcByu2KmpbliCU2AhjcGgrdTwSG4= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -183,6 +192,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -221,8 +231,11 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057 h1:BkX9+SiGBvMou4e1Z+21eTqkqJ8xct8FB+VKzHUTnAY= +github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057/go.mod h1:FGGTNz05swxobKgpWKhnxbEiUUxN+CeHRdF9ViWWPDw= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -262,6 +275,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/test-network-function/test-network-function-claim v1.0.4 h1:f/fT8fKHJifVvJ+koaLXbRzH8HG5TKrkwm67tt+dmY8= github.com/test-network-function/test-network-function-claim v1.0.4/go.mod h1:hDN1y2l8D7K3CX2ZyJThSCBXvVksDLJd1xOMmWqUlaY= +github.com/test-network-function/test-network-function-claim v1.0.5 h1:6MSI0hX/6Z5JTHIxg2n+IQF8qBHAKxvZRdwpdMW8eC8= +github.com/test-network-function/test-network-function-claim v1.0.5/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/pkg/junit/convert.go b/pkg/junit/convert.go index 94a2ebeea..f42361f48 100644 --- a/pkg/junit/convert.go +++ b/pkg/junit/convert.go @@ -29,6 +29,9 @@ const ( // junitContentKey is the "#content" key in the JSON serialized JUnit file. junitContentKey = "#content" + // junitMessageKey is the "-message" key in the JSON serialized JUnit file. + junitMessageKey = "-message" + // junitFailureKey is the "failure" key in the JSON serialized Junit file. junitFailureKey = "failure" @@ -38,9 +41,12 @@ const ( // junitTestNameKey is the "-name" key in the JSON serialized JSON file. junitTestNameKey = "-name" - // junitTestSuiteKey =s the "testsuite" ke in the JSON serialized JSON file. + // junitTestSuiteKey is the "testsuite" key in the JSON serialized JSON file. junitTestSuiteKey = "testsuite" + // junitTestSuitesKey is the "testsuites" key in the JSON serialized JSON file. + junitTestSuitesKey = "testsuites" + // CouldNotDeriveFailureReason is the sentinel message emitted when JUnit failure reason cannot be determined. CouldNotDeriveFailureReason = "could not derive a reason for the failure from the output JSON" ) @@ -88,12 +94,15 @@ func determineFailureReason(failure interface{}) string { var failureReason string if failureReasonObject, ok := failure.(map[string]interface{}); ok { if derivedFailureReason, ok := failureReasonObject[junitContentKey]; ok { - failureReason = derivedFailureReason.(string) + failureReason = "Failed due to line: " + derivedFailureReason.(string) + } else { + failureReason = "Failed due to line: No error line found in JUnit" //nolint:goconst // only instance + } + if derivedFailureReason, ok := failureReasonObject[junitMessageKey]; ok { + failureReason = failureReason + "\n" + "Error message: " + derivedFailureReason.(string) } else { - failureReason = CouldNotDeriveFailureReason + failureReason = failureReason + "\n" + "Error message: No JUinit message found" } - } else { - failureReason = CouldNotDeriveFailureReason } return failureReason } @@ -141,21 +150,25 @@ func toInterfaceArray(object interface{}) []interface{} { func ExtractTestSuiteResults(junitMap map[string]interface{}, reportKeyName string) (map[string]TestResult, error) { // Note: All of the follow checks are paranoia; assuming a well formed JUnit output file, most of these checks // will never fail. As such, individual error reporting per case is ignored in favor of a blanket error statement. - if suite, ok := junitMap[reportKeyName].(map[string]interface{}); ok { - if testSuiteResults, ok := suite[junitTestSuiteKey]; ok { - if testCaseResultsMap, ok := testSuiteResults.(map[string]interface{}); ok { - if testCaseResults, ok := testCaseResultsMap[junitTestCaseKey]; ok { - // Note: order is important since an []interface{} can still cast as interface{}, but an - // interface{} cannot be cast as []interface{} - if resultsObjects, ok := testCaseResults.([]interface{}); ok { - resultsMap := parseResults(resultsObjects) - parseResults(resultsObjects) - return resultsMap, nil + if suites, ok := junitMap[reportKeyName].(map[string]interface{}); ok { + if testSuitesResults, ok := suites[junitTestSuitesKey]; ok { + if testSuitesResultsMap, ok := testSuitesResults.(map[string]interface{}); ok { + if testSuiteResults, ok := testSuitesResultsMap[junitTestSuiteKey]; ok { + if testCaseResultsMap, ok := testSuiteResults.(map[string]interface{}); ok { + if testCaseResults, ok := testCaseResultsMap[junitTestCaseKey]; ok { + // Note: order is important since an []interface{} can still cast as interface{}, but an + // interface{} cannot be cast as []interface{} + if resultsObjects, ok := testCaseResults.([]interface{}); ok { + resultsMap := parseResults(resultsObjects) + parseResults(resultsObjects) + return resultsMap, nil + } + resultsObjects := toInterfaceArray(testCaseResults) + resultsMap := parseResults(resultsObjects) + parseResults(resultsObjects) + return resultsMap, nil + } } - resultsObjects := toInterfaceArray(testCaseResults) - resultsMap := parseResults(resultsObjects) - parseResults(resultsObjects) - return resultsMap, nil } } } diff --git a/pkg/junit/convert_test.go b/pkg/junit/convert_test.go index 3d59892a9..b35e1f207 100644 --- a/pkg/junit/convert_test.go +++ b/pkg/junit/convert_test.go @@ -38,7 +38,7 @@ func TestExtractTestSuiteResults(t *testing.T) { results, err := junit.ExtractTestSuiteResults(claim, testKey) assert.Nil(t, err) // positive test - assert.Equal(t, true, results["generic when Reading namespace of test/test Should not be 'default' and should not begin with 'openshift-'"].Passed) + assert.Equal(t, true, results["[It] operator Runs test on operators operator-install-status-CSV_INSTALLED"].Passed) // negative test - assert.Equal(t, false, results["generic when Testing owners of CNF pod Should contain at least one of kind DaemonSet/ReplicaSet"].Passed, false) + assert.Equal(t, false, results["[It] platform-alteration platform-alteration-boot-params"].Passed) } diff --git a/pkg/junit/testdata/success.junit.xml b/pkg/junit/testdata/success.junit.xml index d37ade7a4..32c6df3ee 100644 --- a/pkg/junit/testdata/success.junit.xml +++ b/pkg/junit/testdata/success.junit.xml @@ -1,21 +1,162 @@ - - - - - - - - - - - - - - /Users/ryangoulding/workspace/t/test-network-function/test-network-function/generic/suite.go:801 Expected <int>: 0 to equal <int>: 1 /Users/ryangoulding/workspace/t/test-network-function/test-network-function/generic/suite.go:808 - - - /Users/ryangoulding/workspace/t/test-network-function/test-network-function/generic/suite.go:500 Expected <int>: 0 to equal <int>: 1 /Users/ryangoulding/workspace/t/test-network-function/test-network-function/generic/suite.go:423 - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/operator/suite.go:86 2021-10-15T11:35:29.914992242-05:00 &{Text:nginx-operator-v0-0-1-sub in namespace tnf Should have a valid subscription Duration:0s} + �[1mSTEP:�[0m nginx-operator-v0-0-1-sub in namespace tnf Should have a valid subscription �[38;5;243m10/15/21 11:35:29.914�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/certification/suite.go:63 2021-10-15T11:35:29.98917388-05:00 &{Text:Getting certification status. Number of containers to check: 1 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/certification/suite.go:70 2021-10-15T11:35:29.989228997-05:00 &{Text:container rhel8/nginx-116 should eventually be verified as certified Duration:0s} + �[1mSTEP:�[0m Getting certification status. Number of containers to check: 1 �[38;5;243m10/15/21 11:35:29.989�[0m �[1mSTEP:�[0m container rhel8/nginx-116 should eventually be verified as certified �[38;5;243m10/15/21 11:35:29.989�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/certification/suite.go:84 2021-10-15T11:35:30.354416591-05:00 &{Text:Verify operator as certified. Number of operators to check: 1 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/certification/suite.go:88 2021-10-15T11:35:30.354554715-05:00 &{Text:should eventually be verified as certified (operator community-operators/etcd) Duration:0s} + �[1mSTEP:�[0m Verify operator as certified. Number of operators to check: 1 �[38;5;243m10/15/21 11:35:30.354�[0m �[1mSTEP:�[0m should eventually be verified as certified (operator community-operators/etcd) �[38;5;243m10/15/21 11:35:30.354�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:212 2021-10-15T11:35:30.471068176-05:00 &{Text:Testing pod nodeSelector Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:217 2021-10-15T11:35:30.472079593-05:00 &{Text:Testing pod nodeSelector tnf/test-697ff58f87-89gmx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:217 2021-10-15T11:35:30.56378219-05:00 &{Text:Testing pod nodeSelector tnf/test-697ff58f87-hfcm6 Duration:0s} + �[1mSTEP:�[0m Testing pod nodeSelector �[38;5;243m10/15/21 11:35:30.471�[0m �[1mSTEP:�[0m Testing pod nodeSelector tnf/test-697ff58f87-89gmx �[38;5;243m10/15/21 11:35:30.472�[0m �[1mSTEP:�[0m Testing pod nodeSelector tnf/test-697ff58f87-hfcm6 �[38;5;243m10/15/21 11:35:30.563�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:234 2021-10-15T11:35:30.634110734-05:00 &{Text:Test terminationGracePeriod Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:239 2021-10-15T11:35:30.634562725-05:00 &{Text:Testing pod terminationGracePeriod tnf test-697ff58f87-89gmx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:239 2021-10-15T11:35:30.704516596-05:00 &{Text:Testing pod terminationGracePeriod tnf test-697ff58f87-hfcm6 Duration:0s} + �[1mSTEP:�[0m Test terminationGracePeriod �[38;5;243m10/15/21 11:35:30.634�[0m �[1mSTEP:�[0m Testing pod terminationGracePeriod tnf test-697ff58f87-89gmx �[38;5;243m10/15/21 11:35:30.634�[0m �[1mSTEP:�[0m Testing pod terminationGracePeriod tnf test-697ff58f87-hfcm6 �[38;5;243m10/15/21 11:35:30.704�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:258 2021-10-15T11:35:30.775579992-05:00 &{Text:Testing PUTs are configured with pre-stop lifecycle Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:262 2021-10-15T11:35:30.775641879-05:00 &{Text:should have pre-stop configured tnf/test-697ff58f87-89gmx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:262 2021-10-15T11:35:30.848140776-05:00 &{Text:should have pre-stop configured tnf/test-697ff58f87-hfcm6 Duration:0s} + �[1mSTEP:�[0m Testing PUTs are configured with pre-stop lifecycle �[38;5;243m10/15/21 11:35:30.775�[0m �[1mSTEP:�[0m should have pre-stop configured tnf/test-697ff58f87-89gmx �[38;5;243m10/15/21 11:35:30.775�[0m �[1mSTEP:�[0m should have pre-stop configured tnf/test-697ff58f87-hfcm6 �[38;5;243m10/15/21 11:35:30.848�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:403 2021-10-15T11:35:30.918743322-05:00 &{Text:Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:409 2021-10-15T11:35:30.918802421-05:00 &{Text:Testing Pod AntiAffinity on Deployment=test, Replicas=2 (ns=tnf) Duration:0s} + �[1mSTEP:�[0m Should set pod replica number greater than 1 and corresponding pod anti-affinity rules in deployment �[38;5;243m10/15/21 11:35:30.918�[0m �[1mSTEP:�[0m Testing Pod AntiAffinity on Deployment=test, Replicas=2 (ns=tnf) �[38;5;243m10/15/21 11:35:30.918�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:295 2021-10-15T11:35:30.993327182-05:00 &{Text:Testing node draining effect of deployment Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:297 2021-10-15T11:35:30.993391098-05:00 &{Text:test deployment in namespace tnf Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:307 2021-10-15T11:35:31.065255957-05:00 &{Text:Should return map of nodes to deployments Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:310 2021-10-15T11:35:31.139364913-05:00 &{Text:should create new replicas when node is drained Duration:0s} + �[1mSTEP:�[0m Testing node draining effect of deployment �[38;5;243m10/15/21 11:35:30.993�[0m �[1mSTEP:�[0m test deployment in namespace tnf �[38;5;243m10/15/21 11:35:30.993�[0m �[1mSTEP:�[0m Should return map of nodes to deployments �[38;5;243m10/15/21 11:35:31.065�[0m �[1mSTEP:�[0m should create new replicas when node is drained �[38;5;243m10/15/21 11:35:31.139�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:180 2021-10-15T11:36:57.660075018-05:00 &{Text:Testing deployment scaling Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:188 2021-10-15T11:36:57.660233946-05:00 &{Text:Scaling Deployment=test, Replicas=2 (ns=tnf) Duration:0s} + �[1mSTEP:�[0m Testing deployment scaling �[38;5;243m10/15/21 11:36:57.66�[0m �[1mSTEP:�[0m Scaling Deployment=test, Replicas=2 (ns=tnf) �[38;5;243m10/15/21 11:36:57.66�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:454 2021-10-15T11:37:06.475755503-05:00 &{Text:Testing owners of CNF pod, should be replicas Set Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:459 2021-10-15T11:37:06.476181458-05:00 &{Text:Should be ReplicaSet tnf test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/lifecycle/suite.go:459 2021-10-15T11:37:06.545811327-05:00 &{Text:Should be ReplicaSet tnf test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Testing owners of CNF pod, should be replicas Set �[38;5;243m10/15/21 11:37:06.475�[0m �[1mSTEP:�[0m Should be ReplicaSet tnf test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:06.476�[0m �[1mSTEP:�[0m Should be ReplicaSet tnf test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:06.545�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:140 2021-10-15T11:37:06.614709622-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx, should not be 'default' or begin with openshift- Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:140 2021-10-15T11:37:06.614764466-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn, should not be 'default' or begin with openshift- Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx, should not be 'default' or begin with openshift- �[38;5;243m10/15/21 11:37:06.614�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn, should not be 'default' or begin with openshift- �[38;5;243m10/15/21 11:37:06.614�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:157 2021-10-15T11:37:06.614925364-05:00 &{Text:Should have a valid ServiceAccount name Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:162 2021-10-15T11:37:06.61532637-05:00 &{Text:Testing pod service account tnf test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:162 2021-10-15T11:37:06.688288844-05:00 &{Text:Testing pod service account tnf test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Should have a valid ServiceAccount name �[38;5;243m10/15/21 11:37:06.614�[0m �[1mSTEP:�[0m Testing pod service account tnf test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:06.615�[0m �[1mSTEP:�[0m Testing pod service account tnf test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:06.688�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:177 2021-10-15T11:37:06.787931993-05:00 &{Text:Should not have RoleBinding in other namespaces Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:183 2021-10-15T11:37:06.788452475-05:00 &{Text:Testing role bidning tnf test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:183 2021-10-15T11:37:06.879475679-05:00 &{Text:Testing role bidning tnf test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Should not have RoleBinding in other namespaces �[38;5;243m10/15/21 11:37:06.787�[0m �[1mSTEP:�[0m Testing role bidning tnf test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:06.788�[0m �[1mSTEP:�[0m Testing role bidning tnf test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:06.879�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:199 2021-10-15T11:37:06.966513448-05:00 &{Text:Should not have ClusterRoleBindings Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:205 2021-10-15T11:37:06.967203987-05:00 &{Text:Testing cluster role bidning tnf test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:205 2021-10-15T11:37:07.087865841-05:00 &{Text:Testing cluster role bidning tnf test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Should not have ClusterRoleBindings �[38;5;243m10/15/21 11:37:06.966�[0m �[1mSTEP:�[0m Testing cluster role bidning tnf test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:06.967�[0m �[1mSTEP:�[0m Testing cluster role bidning tnf test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.087�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.214216335-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.296939624-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:07.214�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.296�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.404501757-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.498824485-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:07.404�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.498�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.582929697-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.655495208-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:07.582�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.655�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.734309905-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.812369266-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:07.734�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.812�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.89551254-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:07.97660132-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:07.895�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:07.976�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.060410972-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.143722032-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:08.06�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:08.143�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.229702179-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.312237822-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:08.229�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:08.312�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.397342461-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.467327258-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:08.397�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:08.467�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.544270833-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/accesscontrol/suite.go:86 2021-10-15T11:37:08.642174686-05:00 &{Text:Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn Duration:0s} + �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:08.544�[0m �[1mSTEP:�[0m Reading namespace of podnamespace= tnf podname= test-697ff58f87-tdwtn �[38;5;243m10/15/21 11:37:08.642�[0m + + + /home/deliedit/redhat/github/david/cnftest/pkg/tnf/test.go:149 github.com/test-network-function/test-network-function/pkg/tnf.(*Test).RunAndValidateWithFailureCallback(0xc000923230, 0x0) /home/deliedit/redhat/github/david/cnftest/pkg/tnf/test.go:149 +0xfa github.com/test-network-function/test-network-function/pkg/tnf.(*Test).RunAndValidate(...) /home/deliedit/redhat/github/david/cnftest/pkg/tnf/test.go:140 github.com/test-network-function/test-network-function/test-network-function/platform.newContainerFsDiffTest(0xc0003831f0, 0xc, 0xc00047e240, 0x0) /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:129 +0x225 github.com/test-network-function/test-network-function/test-network-function/platform.testContainersFsDiff.func1.1() /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:112 +0x1d8 github.com/onsi/ginkgo/internal.(*Suite).runNode.func2(0xc000134000, 0xc0009c2ba0, 0xc0009c2c00, 0xc000341080) /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:724 +0x84 created by github.com/onsi/ginkgo/internal.(*Suite).runNode /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:712 +0x505 + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:111 2021-10-15T11:37:08.739945107-05:00 &{Text:test-697ff58f87-hsqvx(test) should not install new packages after starting Duration:0s} + �[1mSTEP:�[0m test-697ff58f87-hsqvx(test) should not install new packages after starting �[38;5;243m10/15/21 11:37:08.739�[0m + + + /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:299 github.com/test-network-function/test-network-function/test-network-function/platform.testTainted.func1() /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:299 +0x598 github.com/onsi/ginkgo/internal.(*Suite).runNode.func2(0xc000134000, 0xc000609e00, 0xc000609e60, 0xc000e48900) /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:724 +0x84 created by github.com/onsi/ginkgo/internal.(*Suite).runNode /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:712 +0x505 + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:287 2021-10-15T11:37:08.754431552-05:00 &{Text:Testing tainted nodes in cluster Duration:0s} + �[1mSTEP:�[0m Testing tainted nodes in cluster �[38;5;243m10/15/21 11:37:08.754�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:324 2021-10-15T11:37:23.89384141-05:00 &{Text:Should return machineconfig hugepages configuration of node minikube-m03 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:327 2021-10-15T11:37:24.240867623-05:00 &{Text:Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:324 2021-10-15T11:37:29.104091042-05:00 &{Text:Should return machineconfig hugepages configuration of node minikube Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:327 2021-10-15T11:37:29.441875893-05:00 &{Text:Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:324 2021-10-15T11:37:32.539900108-05:00 &{Text:Should return machineconfig hugepages configuration of node minikube-m02 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:327 2021-10-15T11:37:32.868660725-05:00 &{Text:Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. Duration:0s} + �[1mSTEP:�[0m Should return machineconfig hugepages configuration of node minikube-m03 �[38;5;243m10/15/21 11:37:23.893�[0m �[1mSTEP:�[0m Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. �[38;5;243m10/15/21 11:37:24.24�[0m �[1mSTEP:�[0m Should return machineconfig hugepages configuration of node minikube �[38;5;243m10/15/21 11:37:29.104�[0m �[1mSTEP:�[0m Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. �[38;5;243m10/15/21 11:37:29.441�[0m �[1mSTEP:�[0m Should return machineconfig hugepages configuration of node minikube-m02 �[38;5;243m10/15/21 11:37:32.539�[0m �[1mSTEP:�[0m Node's machine config hugepages=0/hugepagesz=2048 values should match the actual ones in the node. �[38;5;243m10/15/21 11:37:32.868�[0m + + + /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:146 github.com/test-network-function/test-network-function/test-network-function/platform.getMcKernelArguments(0xc00067d190, 0xc00002afc0, 0x4, 0xc00002afc0) /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:146 +0x374 github.com/test-network-function/test-network-function/test-network-function/platform.testBootParamsHelper(0xc00067d190, 0xc0002e7a58, 0x15, 0xc0003831e6, 0x3, 0xc00047e240) /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:246 +0x1b9 github.com/test-network-function/test-network-function/test-network-function/platform.testBootParams.func1() /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:238 +0xd7 github.com/onsi/ginkgo/internal.(*Suite).runNode.func2(0xc000134000, 0xc000ec0ba0, 0xc000ec0c00, 0xc000eea480) /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:724 +0x84 created by github.com/onsi/ginkgo/internal.(*Suite).runNode /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:712 +0x505 + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:243 2021-10-15T11:37:37.276527244-05:00 &{Text:Testing boot params for the pod's node tnf/test-697ff58f87-hsqvx Duration:0s} + �[1mSTEP:�[0m Testing boot params for the pod's node tnf/test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:37.276�[0m + + + /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:146 github.com/test-network-function/test-network-function/test-network-function/platform.getMcKernelArguments(0xc00073c820, 0xc0003fa030, 0x4, 0xc0003fa030) /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:146 +0x374 github.com/test-network-function/test-network-function/test-network-function/platform.testSysctlConfigsHelper(0xc0002e7a58, 0x15, 0xc0003831e6, 0x3) /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:276 +0x1e5 github.com/test-network-function/test-network-function/test-network-function/platform.testSysctlConfigs.func1() /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:266 +0xbf github.com/onsi/ginkgo/internal.(*Suite).runNode.func2(0xc000134000, 0xc000e604e0, 0xc000e60540, 0xc000254780) /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:724 +0x84 created by github.com/onsi/ginkgo/internal.(*Suite).runNode /home/deliedit/go/pkg/mod/github.com/onsi/ginkgo@v1.16.6-0.20211014152641-f228134fe057/internal/suite.go:712 +0x505 + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:271 2021-10-15T11:37:37.753637946-05:00 &{Text:Testing sysctl config files for the pod's node tnf/test-697ff58f87-hsqvx Duration:0s} + �[1mSTEP:�[0m Testing sysctl config files for the pod's node tnf/test-697ff58f87-hsqvx �[38;5;243m10/15/21 11:37:37.753�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:81 2021-10-15T11:37:40.726721736-05:00 &{Text:should report a proper Red Hat version Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:93 2021-10-15T11:37:40.726784282-05:00 &{Text:test-697ff58f87-hsqvx(test) is checked for Red Hat version Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/platform/suite.go:93 2021-10-15T11:37:40.734002808-05:00 &{Text:test-697ff58f87-tdwtn(test) is checked for Red Hat version Duration:0s} + �[1mSTEP:�[0m should report a proper Red Hat version �[38;5;243m10/15/21 11:37:40.726�[0m �[1mSTEP:�[0m test-697ff58f87-hsqvx(test) is checked for Red Hat version �[38;5;243m10/15/21 11:37:40.726�[0m �[1mSTEP:�[0m test-697ff58f87-tdwtn(test) is checked for Red Hat version �[38;5;243m10/15/21 11:37:40.734�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:88 2021-10-15T11:37:40.738043989-05:00 &{Text:a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-hsqvx(test) 10.244.2.179 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:92 2021-10-15T11:37:44.854881652-05:00 &{Text:a Ping is issued from test-697ff58f87-hsqvx(test) to partner-68cf756959-sczv6(partner) 10.244.2.183 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:88 2021-10-15T11:37:48.998931451-05:00 &{Text:a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-tdwtn(test) 10.244.0.79 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:92 2021-10-15T11:37:53.054891245-05:00 &{Text:a Ping is issued from test-697ff58f87-tdwtn(test) to partner-68cf756959-sczv6(partner) 10.244.2.183 Duration:0s} + �[1mSTEP:�[0m a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-hsqvx(test) 10.244.2.179 �[38;5;243m10/15/21 11:37:40.738�[0m �[1mSTEP:�[0m a Ping is issued from test-697ff58f87-hsqvx(test) to partner-68cf756959-sczv6(partner) 10.244.2.183 �[38;5;243m10/15/21 11:37:44.854�[0m �[1mSTEP:�[0m a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-tdwtn(test) 10.244.0.79 �[38;5;243m10/15/21 11:37:48.998�[0m �[1mSTEP:�[0m a Ping is issued from test-697ff58f87-tdwtn(test) to partner-68cf756959-sczv6(partner) 10.244.2.183 �[38;5;243m10/15/21 11:37:53.054�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:114 2021-10-15T11:37:57.155974016-05:00 &{Text:a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-hsqvx(test) 10.244.2.179 Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:114 2021-10-15T11:38:01.295628767-05:00 &{Text:a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-tdwtn(test) 10.244.0.79 Duration:0s} + �[1mSTEP:�[0m a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-hsqvx(test) 10.244.2.179 �[38;5;243m10/15/21 11:37:57.155�[0m �[1mSTEP:�[0m a Ping is issued from partner-68cf756959-sczv6(partner) to test-697ff58f87-tdwtn(test) 10.244.0.79 �[38;5;243m10/15/21 11:38:01.295�[0m + + + Report Entries: By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:142 2021-10-15T11:38:05.328059396-05:00 &{Text:Testing services in namespace tnf Duration:0s} -- By Step /home/deliedit/redhat/github/david/cnftest/test-network-function/networking/suite.go:142 2021-10-15T11:38:05.411991173-05:00 &{Text:Testing services in namespace tnf Duration:0s} + �[1mSTEP:�[0m Testing services in namespace tnf �[38;5;243m10/15/21 11:38:05.328�[0m �[1mSTEP:�[0m Testing services in namespace tnf �[38;5;243m10/15/21 11:38:05.411�[0m + + + + + + + + \ No newline at end of file diff --git a/pkg/tnf/handlers/generic/assertion/assertion.go b/pkg/tnf/handlers/generic/assertion/assertion.go index e0f3fd6fe..dbb69b087 100644 --- a/pkg/tnf/handlers/generic/assertion/assertion.go +++ b/pkg/tnf/handlers/generic/assertion/assertion.go @@ -150,13 +150,9 @@ func (a *Assertion) UnmarshalJSON(b []byte) error { return err } - if err := a.unmarshalGroupIdxJSON(objMap); err != nil { + if err = a.unmarshalGroupIdxJSON(objMap); err != nil { return err } - if err := a.unmarshalConditionJSON(objMap); err != nil { - return err - } - - return nil + return a.unmarshalConditionJSON(objMap) } diff --git a/pkg/tnf/identifier/identifier.go b/pkg/tnf/identifier/identifier.go index daae5a9b9..0434e328d 100644 --- a/pkg/tnf/identifier/identifier.go +++ b/pkg/tnf/identifier/identifier.go @@ -79,13 +79,9 @@ func (i *Identifier) UnmarshalJSON(b []byte) error { return err } - if err := i.unmarshalURL(objMap); err != nil { + if err = i.unmarshalURL(objMap); err != nil { return err } - if err := i.unmarshalSemanticVersion(objMap); err != nil { - return err - } - - return nil + return i.unmarshalSemanticVersion(objMap) } diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index ed2cf9405..40e5bb0ca 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +set -x # defaults export OUTPUT_LOC="$PWD/test-network-function" @@ -72,4 +72,8 @@ echo "Running with focus '$FOCUS'" echo "Running with skip '$SKIP'" echo "Report will be output to '$OUTPUT_LOC'" echo "ginkgo arguments '${GINKGO_ARGS}'" -cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" -ginkgo.skip="$SKIP" ${GINKGO_ARGS} +SKIP_STRING="" +if [ -z $SKIP ]; then + $SKIP_STRING=-ginkgo.skip="$SKIP" +fi +cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" $SKIP_STRING ${GINKGO_ARGS} diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 817dc51cb..fb3716b91 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config" @@ -38,7 +37,8 @@ import ( ) var _ = ginkgo.Describe(common.AccessControlTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AccessControlTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.AccessControlTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() @@ -46,6 +46,8 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) + ginkgo.ReportAfterEach(results.RecordResult) + testNamespace(env) testRoles(env) @@ -75,7 +77,7 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { //nolint:gocritic,funlen // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, testType string) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHostResourceIdentifier) + "-" + testCmd.Name + testID := identifiers.XformToGinkgoItIdentifierExtended(identifiers.TestHostResourceIdentifier, testCmd.Name) ginkgo.It(testID, func() { context := common.GetContext() for _, podUnderTest := range env.PodsUnderTest { @@ -87,7 +89,6 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, testCmd.ExpectedStatusFn(podName, testcases.StatusFunctionType(val)) } } - defer results.RecordResult(identifiers.TestHostResourceIdentifier) testType := testType testCmd := testCmd var args []interface{} @@ -137,7 +138,6 @@ func testNamespace(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s, should not be 'default' or begin with openshift-", podNamespace, podName)) - defer results.RecordResult(identifiers.TestNamespaceBestPracticesIdentifier) gomega.Expect(podNamespace).To(gomega.Not(gomega.Equal("default"))) gomega.Expect(podNamespace).To(gomega.Not(gomega.HavePrefix("openshift-"))) } @@ -160,7 +160,6 @@ func testServiceAccount(env *config.TestEnvironment) { podNamespace := podUnderTest.Namespace context := common.GetContext() ginkgo.By(fmt.Sprintf("Testing pod service account %s %s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestPodServiceAccountBestPracticesIdentifier) tester := serviceaccount.NewServiceAccount(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -181,7 +180,6 @@ func testRoleBindings(env *config.TestEnvironment) { podNamespace := podUnderTest.Namespace serviceAccountName := podUnderTest.ServiceAccount context := common.GetContext() - defer results.RecordResult(identifiers.TestPodRoleBindingsBestPracticesIdentifier) ginkgo.By(fmt.Sprintf("Testing role bidning %s %s", podNamespace, podName)) if serviceAccountName == "" { ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") @@ -204,7 +202,6 @@ func testClusterRoleBindings(env *config.TestEnvironment) { podNamespace := podUnderTest.Namespace serviceAccountName := podUnderTest.ServiceAccount context := common.GetContext() - defer results.RecordResult(identifiers.TestPodClusterRoleBindingsBestPracticesIdentifier) ginkgo.By(fmt.Sprintf("Testing cluster role bidning %s %s", podNamespace, podName)) if serviceAccountName == "" { ginkgo.Skip("Can not test when serviceAccountName is empty. Please check previous tests for failures") diff --git a/test-network-function/certification/suite.go b/test-network-function/certification/suite.go index 5fb341160..0452e8cbe 100644 --- a/test-network-function/certification/suite.go +++ b/test-network-function/certification/suite.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/internal/api" configpkg "github.com/test-network-function/test-network-function/pkg/config" @@ -40,12 +39,15 @@ const ( var certAPIClient api.CertAPIClient var _ = ginkgo.Describe(common.AffiliatedCertTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.AffiliatedCertTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.AffiliatedCertTestKey) { env := configpkg.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() }) + ginkgo.ReportAfterEach(results.RecordResult) + testContainerCertificationStatus() testOperatorCertificationStatus() } @@ -59,7 +61,6 @@ func testContainerCertificationStatus() { cnfsToQuery := env.Config.CertifiedContainerInfo ginkgo.By(fmt.Sprintf("Getting certification status. Number of containers to check: %d", len(cnfsToQuery))) - defer results.RecordResult(identifiers.TestContainerIsCertifiedIdentifier) if len(cnfsToQuery) > 0 { certAPIClient = api.NewHTTPClient() @@ -81,7 +82,6 @@ func testOperatorCertificationStatus() { ginkgo.It(testID, func() { operatorsToQuery := configpkg.GetTestEnvironment().Config.CertifiedOperatorInfo ginkgo.By(fmt.Sprintf("Verify operator as certified. Number of operators to check: %d", len(operatorsToQuery))) - defer results.RecordResult(identifiers.TestOperatorIsCertifiedIdentifier) if len(operatorsToQuery) > 0 { certAPIClient := api.NewHTTPClient() for _, certified := range operatorsToQuery { diff --git a/test-network-function/common/suite.go b/test-network-function/common/suite.go index ea6269491..50a4f6ff6 100644 --- a/test-network-function/common/suite.go +++ b/test-network-function/common/suite.go @@ -23,23 +23,21 @@ import ( "github.com/test-network-function/test-network-function/pkg/config/autodiscover" ) -var _ = ginkgo.Describe(CommonTestKey, func() { - var env *configpkg.TestEnvironment +var env *configpkg.TestEnvironment - ginkgo.BeforeSuite(func() { - for name := range autodiscover.GetNodesList() { - autodiscover.DeleteDebugLabel(name) - } - env = configpkg.GetTestEnvironment() - env.LoadAndRefresh() - }) +var _ = ginkgo.BeforeSuite(func() { + for name := range autodiscover.GetNodesList() { + autodiscover.DeleteDebugLabel(name) + } + env = configpkg.GetTestEnvironment() + env.LoadAndRefresh() +}) - ginkgo.AfterSuite(func() { - // clean up added label to nodes - log.Info("clean up added labels to nodes") - for name, node := range env.NodesUnderTest { - node.Oc.Close() - autodiscover.DeleteDebugLabel(name) - } - }) +var _ = ginkgo.AfterSuite(func() { + // clean up added label to nodes + log.Info("clean up added labels to nodes") + for name, node := range env.NodesUnderTest { + node.Oc.Close() + autodiscover.DeleteDebugLabel(name) + } }) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index f2a35fecc..f7260a07d 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -9,10 +9,8 @@ import ( "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/clusterversion" @@ -20,6 +18,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodedebug" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/test-network-function/results" ) const ( @@ -66,71 +65,56 @@ var ( ) var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.DiagnosticTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.DiagnosticTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) - ginkgo.When("a cluster is set up and installed with OpenShift", func() { - env := config.GetTestEnvironment() - ginkgo.BeforeEach(func() { - env.LoadAndRefresh() - gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) - gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) - }) - ginkgo.By("should report OCP version") - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestclusterVersionIdentifier) - ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestclusterVersionIdentifier) - testOcpVersion() - }) - - testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestExtractNodeInformationIdentifier) - ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestExtractNodeInformationIdentifier) - context := common.GetContext() - - tester, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(jsonParseResult).ToNot(gomega.BeNil()) - gomega.Expect(jsonParseResult.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - - test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(test).ToNot(gomega.BeNil()) - - test.RunAndValidate() - - genericTest := (*tester).(*generic.Generic) - gomega.Expect(genericTest).ToNot(gomega.BeNil()) - matches := genericTest.Matches - gomega.Expect(len(matches)).To(gomega.Equal(1)) - match := genericTest.GetMatches()[0] - err = json.Unmarshal([]byte(match.Match), &nodeSummary) - gomega.Expect(err).To(gomega.BeNil()) - }) - ginkgo.By("should report all CNI plugins") - testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestListCniPluginsIdentifier) - ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestListCniPluginsIdentifier) - testCniPlugins() - }) - ginkgo.By("should report nodes HW info") - testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestNodesHwInfoIdentifier) - ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestNodesHwInfoIdentifier) - testNodesHwInfo() - }) - ginkgo.By("should report cluster CSI driver info") - testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestClusterCsiInfoIdentifier) - ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestClusterCsiInfoIdentifier) - listClusterCSIInfo() - }) + ginkgo.ReportAfterEach(results.RecordResult) + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestclusterVersionIdentifier) + ginkgo.It(testID, func() { + testOcpVersion() + }) + + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestExtractNodeInformationIdentifier) + ginkgo.It(testID, func() { + context := common.GetContext() + + tester, handlers, jsonParseResult, err := generic.NewGenericFromJSONFile(relativeNodesTestPath, relativeSchemaPath) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(jsonParseResult).ToNot(gomega.BeNil()) + gomega.Expect(jsonParseResult.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + + test.RunAndValidate() + + genericTest := (*tester).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + match := genericTest.GetMatches()[0] + err = json.Unmarshal([]byte(match.Match), &nodeSummary) + gomega.Expect(err).To(gomega.BeNil()) + }) + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestListCniPluginsIdentifier) + ginkgo.It(testID, func() { + testCniPlugins() + }) + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestNodesHwInfoIdentifier) + ginkgo.It(testID, func() { + testNodesHwInfo() + }) + testID = identifiers.XformToGinkgoItIdentifier(identifiers.TestClusterCsiInfoIdentifier) + ginkgo.It(testID, func() { + listClusterCSIInfo() }) } }) diff --git a/test-network-function/generic/suite.go b/test-network-function/generic/suite.go index a12363db2..7699e0569 100644 --- a/test-network-function/generic/suite.go +++ b/test-network-function/generic/suite.go @@ -17,11 +17,10 @@ package generic import ( + "github.com/onsi/ginkgo" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" - - "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" + "github.com/test-network-function/test-network-function/test-network-function/results" ) const ( @@ -30,10 +29,12 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(testsKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testsKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, testsKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() }) + ginkgo.ReportAfterEach(results.RecordResult) } }) diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 06e371430..63b6dfc60 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -56,6 +56,9 @@ func formTestURL(suite, name string) string { } var ( + // TestIdToClaimId converts the testcase short ID to the claim identifier + TestIDToClaimID = map[string]claim.Identifier{} + // TestHostResourceIdentifier tests container best practices. TestHostResourceIdentifier = claim.Identifier{ Url: formTestURL(common.AccessControlTestKey, "host-resource"), @@ -223,7 +226,21 @@ func formDescription(identifier claim.Identifier, description string) string { // XformToGinkgoItIdentifier transform the claim.Identifier into a test Id that can be used to skip // specific tests func XformToGinkgoItIdentifier(identifier claim.Identifier) string { - return strings.ReplaceAll(strings.TrimPrefix(identifier.Url, url+"/"), "/", "-") + return XformToGinkgoItIdentifierExtended(identifier, "") +} + +// XformToGinkgoItIdentifierExtended transform the claim.Identifier into a test Id that can be used to skip +// specific tests +func XformToGinkgoItIdentifierExtended(identifier claim.Identifier, extra string) string { + itID := strings.ReplaceAll(strings.TrimPrefix(identifier.Url, url+"/"), "/", "-") + var key string + if extra != "" { + key = itID + "-" + extra + } else { + key = itID + } + TestIDToClaimID[key] = identifier + return key } // Catalog is the JUnit testcase catalog of tests. diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 093a426a6..11a75718e 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -30,10 +30,8 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -43,6 +41,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodeselector" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/owners" "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/test-network-function/test-network-function/test-network-function/results" ) const ( @@ -84,7 +83,8 @@ var drainTimeout = time.Duration(drainTimeoutMinutes) * time.Minute // All actual test code belongs below here. Utilities belong above. // var _ = ginkgo.Describe(common.LifecycleTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.LifecycleTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.LifecycleTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() @@ -93,6 +93,8 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { }) + ginkgo.ReportAfterEach(results.RecordResult) + testNodeSelector(env) testGracePeriod(env) @@ -177,7 +179,6 @@ func testScaling(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestScalingIdentifier) ginkgo.It(testID, func() { ginkgo.By("Testing deployment scaling") - defer results.RecordResult(identifiers.TestScalingIdentifier) defer restoreDeployments(env) defer env.SetNeedsRefresh() @@ -211,7 +212,6 @@ func testNodeSelector(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing pod nodeSelector %s/%s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestPodNodeSelectorAndAffinityBestPractices) infoWriter := tnf.CreateTestExtraInfoWriter() tester := nodeselector.NewNodeSelector(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) @@ -227,14 +227,13 @@ func testNodeSelector(env *config.TestEnvironment) { func testGracePeriod(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNonDefaultGracePeriodIdentifier) - ginkgo.It(testID+" ", func() { + ginkgo.It(testID, func() { ginkgo.By("Test terminationGracePeriod") context := common.GetContext() for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing pod terminationGracePeriod %s %s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestNonDefaultGracePeriodIdentifier) infoWriter := tnf.CreateTestExtraInfoWriter() tester := graceperiod.NewGracePeriod(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) @@ -258,7 +257,6 @@ func testShutdown(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("should have pre-stop configured %s/%s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestShudtownIdentifier) shutdownTest(podNamespace, podName) } }) @@ -291,8 +289,6 @@ func testPodsRecreation(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRecreationIdentifier) ginkgo.It(testID, func() { ginkgo.By("Testing node draining effect of deployment") - defer results.RecordResult(identifiers.TestPodRecreationIdentifier) - ginkgo.By(fmt.Sprintf("test deployment in namespace %s", env.NameSpaceUnderTest)) deployments, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) if len(deployments) == 0 { @@ -382,7 +378,6 @@ func testPodAntiAffinity(env *config.TestEnvironment) { } for _, deployment := range env.DeploymentsUnderTest { - defer results.RecordResult(identifiers.TestPodHighAvailabilityBestPractices) ginkgo.By(fmt.Sprintf("Testing Pod AntiAffinity on Deployment=%s, Replicas=%d (ns=%s)", deployment.Name, deployment.Replicas, deployment.Namespace)) podAntiAffinity(deployment.Name, deployment.Namespace, deployment.Replicas) @@ -434,7 +429,6 @@ func testOwner(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Should be ReplicaSet %s %s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestPodDeploymentBestPracticesIdentifier) tester := owners.NewOwners(common.DefaultTimeout, podNamespace, podName) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 4bd931b7a..15a379390 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -24,10 +24,8 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf" @@ -35,6 +33,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/ping" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/test-network-function/test-network-function/test-network-function/results" ) const ( @@ -47,13 +46,17 @@ const ( // Runs the "generic" CNF test cases. var _ = ginkgo.Describe(common.NetworkingTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.NetworkingTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.NetworkingTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) + + ginkgo.ReportAfterEach(results.RecordResult) + ginkgo.Context("Both Pods are on the Default network", func() { // for each container under test, ensure bidirectional ICMP traffic between the container and the orchestrator. testDefaultNetworkConnectivity(env, defaultNumPings) @@ -85,7 +88,6 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), cut.DefaultNetworkIPAddress)) - defer results.RecordResult(identifiers.TestICMPv4ConnectivityIdentifier) testPing(testOrchestrator.Oc, cut.DefaultNetworkIPAddress, count) ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), @@ -112,7 +114,6 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), cut.DefaultNetworkIPAddress)) - defer results.RecordResult(identifiers.TestICMPv4ConnectivityIdentifier) testPing(testOrchestrator.Oc, multusIPAddress, count) } } @@ -135,7 +136,6 @@ func testPing(initiatingPodOc *interactive.Oc, targetPodIPAddress string, count func testNodePort(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestServicesDoNotUseNodeportsIdentifier) ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestServicesDoNotUseNodeportsIdentifier) context := common.GetContext() ginkgo.By(fmt.Sprintf("Testing services in namespace %s", env.NameSpaceUnderTest)) tester := nodeport.NewNodePort(common.DefaultTimeout, env.NameSpaceUnderTest) diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 5d172e0a4..71f7fe2f7 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -22,7 +22,6 @@ import ( "time" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" @@ -52,13 +51,15 @@ var ( ) var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.ObservabilityTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.ObservabilityTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) + ginkgo.ReportAfterEach(results.RecordResult) testLogging(env) testCrds(env) } @@ -69,7 +70,6 @@ func testLogging(env *config.TestEnvironment) { ginkgo.It(testID, func() { for _, cut := range env.ContainersUnderTest { ginkgo.By(fmt.Sprintf("Test container: %+v. should emit at least one line of log to stderr/stdout", cut.ContainerIdentifier)) - defer results.RecordResult(identifiers.TestLoggingIdentifier) loggingTest(cut.ContainerIdentifier) } }) diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 732e706f5..754ccb01e 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -25,17 +25,15 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/operator" - "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/test-network-function/results" ) const ( @@ -45,8 +43,6 @@ const ( ) var ( - context *interactive.Context - err error // checkSubscriptionTestPath is the file location of the uncordon.json test case relative to the project root. checkSubscriptionTestPath = path.Join("pkg", "tnf", "handlers", "checksubscription", "check-subscription.json") @@ -65,7 +61,8 @@ var ( ) var _ = ginkgo.Describe(testSpecName, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, testSpecName) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, testSpecName) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() @@ -73,16 +70,8 @@ var _ = ginkgo.Describe(testSpecName, func() { ginkgo.Skip("No Operator found.") } }) - + ginkgo.ReportAfterEach(results.RecordResult) defer ginkgo.GinkgoRecover() - ginkgo.When("a local shell is spawned", func() { - context = common.GetContext() - ginkgo.It("should be created without error", func() { - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(context).ToNot(gomega.BeNil()) - gomega.Expect(context.GetExpecter()).ToNot(gomega.BeNil()) - }) - }) ginkgo.Context("Runs test on operators", func() { itRunsTestsOnOperator(env) }) @@ -95,7 +84,6 @@ func testOperatorsAreInstalledViaOLM(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorIsInstalledViaOLMIdentifier) ginkgo.It(testID, func() { for _, operatorInTest := range env.OperatorsUnderTest { - defer results.RecordResult(identifiers.TestOperatorIsInstalledViaOLMIdentifier) ginkgo.By(fmt.Sprintf("%s in namespace %s Should have a valid subscription", operatorInTest.SubscriptionName, operatorInTest.Namespace)) testOperatorIsInstalledViaOLM(operatorInTest.SubscriptionName, operatorInTest.Namespace) } @@ -114,7 +102,7 @@ func testOperatorIsInstalledViaOLM(subscriptionName, subscriptionNamespace strin gomega.Expect(handlers).ToNot(gomega.BeNil()) gomega.Expect(len(handlers)).To(gomega.Equal(1)) gomega.Expect(tester).ToNot(gomega.BeNil()) - + context := common.GetContext() test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) @@ -142,9 +130,8 @@ func itRunsTestsOnOperator(env *config.TestEnvironment) { //nolint:gocritic // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. func runTestsOnOperator(env *config.TestEnvironment, testCase testcases.BaseTestCase) { - testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestOperatorInstallStatusIdentifier) + "-" + testCase.Name + testID := identifiers.XformToGinkgoItIdentifierExtended(identifiers.TestOperatorInstallStatusIdentifier, testCase.Name) ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestOperatorInstallStatusIdentifier) for _, op := range env.OperatorsUnderTest { if testCase.ExpectedType == testcases.Function { for _, val := range testCase.ExpectedStatus { @@ -156,6 +143,7 @@ func runTestsOnOperator(env *config.TestEnvironment, testCase testcases.BaseTest cmdArgs := strings.Split(fmt.Sprintf(testCase.Command, args...), " ") opInTest := operator.NewOperator(cmdArgs, name, op.Namespace, testCase.ExpectedStatus, testCase.ResultType, testCase.Action, common.DefaultTimeout) gomega.Expect(opInTest).ToNot(gomega.BeNil()) + context := common.GetContext() test, err := tnf.NewTest(context.GetExpecter(), opInTest, []reel.Handler{opInTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 4c85370fb..b4d28bbe6 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -30,10 +30,8 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" - "github.com/test-network-function/test-network-function/test-network-function/results" "github.com/onsi/ginkgo" - ginkgoconfig "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/base/redhat" @@ -51,6 +49,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" utils "github.com/test-network-function/test-network-function/pkg/utils" + "github.com/test-network-function/test-network-function/test-network-function/results" ) // @@ -80,13 +79,15 @@ func getTaintedBitValues() []string { } var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { - if testcases.IsInFocus(ginkgoconfig.GinkgoConfig.FocusStrings, common.PlatformAlterationTestKey) { + conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.PlatformAlterationTestKey) { env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) + ginkgo.ReportAfterEach(results.RecordResult) // use this boolean to turn off tests that require OS packages if !common.IsMinikube() { testContainersFsDiff(env) @@ -104,7 +105,6 @@ func testIsRedHatRelease(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestIsRedHatReleaseIdentifier) ginkgo.It(testID, func() { ginkgo.By("should report a proper Red Hat version") - defer results.RecordResult(identifiers.TestIsRedHatReleaseIdentifier) for _, cut := range env.ContainersUnderTest { testContainerIsRedHatRelease(cut) } @@ -149,7 +149,6 @@ func testContainersFsDiff(env *config.TestEnvironment) { // newContainerFsDiffTest test that the CUT didn't install new packages after starting, and report through Ginkgo. func newContainerFsDiffTest(nodeName string, nodeOc, targetContainerOC *interactive.Oc) *tnf.Test { - defer results.RecordResult(identifiers.TestUnalteredBaseImageIdentifier) targetContainerOC.GetExpecter() containerIDTester := containerid.NewContainerID(common.DefaultTimeout) test, err := tnf.NewTest(targetContainerOC.GetExpecter(), containerIDTester, []reel.Handler{containerIDTester}, targetContainerOC.GetErrorChannel()) @@ -267,7 +266,6 @@ func testBootParams(env *config.TestEnvironment) { } func testBootParamsHelper(context *interactive.Context, podName, podNamespace string, targetContainerOc *interactive.Oc) { ginkgo.By(fmt.Sprintf("Testing boot params for the pod's node %s/%s", podNamespace, podName)) - defer results.RecordResult(identifiers.TestUnalteredStartupBootParamsIdentifier) nodeName := getPodNodeName(context, podName, podNamespace) mcName := getMcName(context, nodeName) mcKernelArgumentsMap := getMcKernelArguments(context, mcName) @@ -287,7 +285,8 @@ func testBootParamsHelper(context *interactive.Context, podName, podNamespace st } func testSysctlConfigs(env *config.TestEnvironment) { - ginkgo.It("platform-sysctl-config", func() { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestSysctlConfigsIdentifier) + ginkgo.It(testID, func() { for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNameSpace := podUnderTest.Namespace @@ -370,8 +369,6 @@ func getNodeMcHugepages(nodeName string) (hugePagesCount, hugePagesSize int) { func testHugepages(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) ginkgo.It(testID, func() { - defer results.RecordResult(identifiers.TestHugepagesNotManuallyManipulated) - var badNodes []string for _, node := range env.NodesUnderTest { if !node.IsWorker() { diff --git a/test-network-function/results/results.go b/test-network-function/results/results.go index 399c29b3a..434fd2a4a 100644 --- a/test-network-function/results/results.go +++ b/test-network-function/results/results.go @@ -18,47 +18,57 @@ package results import ( "fmt" + "strings" - "github.com/onsi/ginkgo" + ginkgoTypes "github.com/onsi/ginkgo/types" "github.com/test-network-function/test-network-function-claim/pkg/claim" - "github.com/test-network-function/test-network-function/pkg/junit" + "github.com/test-network-function/test-network-function/test-network-function/identifiers" ) -var results = map[claim.Identifier][]claim.Result{} +// results is the results map +var results = map[string][]claim.Result{} // RecordResult is a hook provided to save aspects of the ginkgo.GinkgoTestDescription for a given claim.Identifier. // Multiple results for a given identifier are aggregated as an array under the same key. -func RecordResult(identifier claim.Identifier) { - testContext := ginkgo.CurrentGinkgoTestDescription() - results[identifier] = append(results[identifier], claim.Result{ - Duration: int(testContext.Duration.Nanoseconds()), - Filename: testContext.FileName, - IsMeasurement: testContext.IsMeasurement, - LineNumber: testContext.LineNumber, - TestText: testContext.FullTestText, - }) +func RecordResult(report ginkgoTypes.SpecReport) { //nolint:gocritic // From Ginkgo + if claimID, ok := identifiers.TestIDToClaimID[report.LeafNodeText]; ok { + var key string + for _, level := range report.ContainerHierarchyTexts { + levelNoSpace := strings.ReplaceAll(level, " ", "_") + key = key + "-" + levelNoSpace + } + key = strings.TrimLeft(key, "-") + "-" + report.LeafNodeText + testText := identifiers.Catalog[claimID].Description + results[key] = append(results[key], claim.Result{ + Duration: int(report.RunTime.Nanoseconds()), + FailureLocation: report.FailureLocation().String(), + FailureLineContent: report.FailureLocation().ContentsOfLine(), + TestText: testText, + FailureReason: report.FailureMessage(), + State: report.State.String(), + StartTime: report.StartTime.String(), + EndTime: report.EndTime.String(), + CapturedTestOutput: report.CapturedGinkgoWriterOutput, + TestID: &claimID, + }) + } else { + panic(fmt.Sprintf("TestID %s has no corresponding Claim ID", report.LeafNodeText)) + } } // GetReconciledResults is a function added to aggregate a Claim's results. Due to the limitations of // test-network-function-claim's Go Client, results are generalized to map[string]interface{}. This method is needed // to take the results gleaned from JUnit output, and to combine them with the contexts built up by subsequent calls to // RecordResult. The combination of the two forms a Claim's results. -func GetReconciledResults(testResults map[string]junit.TestResult) map[string]interface{} { +func GetReconciledResults() map[string]interface{} { resultMap := make(map[string]interface{}) for key, vals := range results { - // JSON cannot handle complex key types, so this flattens the complex key into a string format. - strKey := fmt.Sprintf("{\"url\":\"%s\",\"version\":\"%s\"}", key.Url, key.Version) // initializes the result map, if necessary - if _, ok := resultMap[strKey]; !ok { - resultMap[strKey] = make([]claim.Result, 0) + if _, ok := resultMap[key]; !ok { + resultMap[key] = make([]claim.Result, 0) } - // a codec which correlates claim.Result, JUnit results (testResults), and builds up the map - // of claim's results. - for _, val := range vals { - val.Passed = testResults[val.TestText].Passed - testFailReason := testResults[val.TestText].FailureReason - val.FailureReason = testFailReason - resultMap[strKey] = append(resultMap[strKey].([]claim.Result), val) + for _, val := range vals { //nolint:gocritic // Only done once at the end + resultMap[key] = append(resultMap[key].([]claim.Result), val) } } return resultMap diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 1df89ebcc..edb047e91 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -123,7 +123,8 @@ func TestTest(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) common.SetLogFormat() common.SetLogLevel() - + // Display GinkGo Version + log.Info("Ginkgo Version: ", ginkgo.GINKGO_VERSION) // Display the latest previously released build in case this build is not released // Otherwise display the build version if GitRelease == "" { @@ -156,8 +157,7 @@ func TestTest(t *testing.T) { // fill out the remaining claim information. claimData.RawResults = junitMap - resultMap := generateResultMap(junitMap) - claimData.Results = results.GetReconciledResults(resultMap) + claimData.Results = results.GetReconciledResults() configurations := marshalConfigurations() claimData.Nodes = generateNodes() unmarshalConfigurations(configurations, claimData.Configurations) @@ -180,16 +180,6 @@ func incorporateVersions(claimData *claim.Claim) { } } -// generateResultMap is a conversion utility to generate results. If an error is encountered, than this method fails -// fatally. -func generateResultMap(junitMap map[string]interface{}) map[string]junit.TestResult { - resultMap, err := junit.ExtractTestSuiteResults(junitMap, TNFReportKey) - if err != nil { - log.Fatalf("Could not extract the test suite results: %s", err) - } - return resultMap -} - // appendCNFFeatureValidationReportResults is a helper method to add the results of running the cnf-features-deploy // test suite to the claim file. func appendCNFFeatureValidationReportResults(junitPath *string, junitMap map[string]interface{}) { From d2818cbe2149aa4f60287b1fb526ddfdbae5dc4c Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 27 Oct 2021 23:12:18 -0400 Subject: [PATCH 110/344] make sure node has debug pod before testing tainted kernel (#414) --- test-network-function/platform/suite.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index b4d28bbe6..f18e06919 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -329,6 +329,9 @@ func testTainted(env *config.TestEnvironment) { var taintedNodes []string for _, node := range env.NodesUnderTest { + if !node.HasDebugPod() { + continue + } context := node.Oc tester := nodetainted.NewNodeTainted(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) @@ -371,7 +374,7 @@ func testHugepages(env *config.TestEnvironment) { ginkgo.It(testID, func() { var badNodes []string for _, node := range env.NodesUnderTest { - if !node.IsWorker() { + if !node.IsWorker() || !node.HasDebugPod() { continue } nodeOc := node.Oc From ffbb611d601b0a739a1a1872e07f1d879e156ff2 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 29 Oct 2021 15:28:56 +0200 Subject: [PATCH 111/344] Fix observability TS panic in CRD TC. (#421) --- test-network-function/observability/suite.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 71f7fe2f7..969bee18f 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -96,11 +96,10 @@ func loggingTest(c configsections.ContainerIdentifier) { } func testCrds(env *config.TestEnvironment) { - ginkgo.By("CRDs should have a status subresource") testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestCrdsStatusSubresourceIdentifier) ginkgo.It(testID, func() { + ginkgo.By("CRDs should have a status subresource") context := common.GetContext() - for _, crdName := range env.CrdNames { ginkgo.By("Testing CRD " + crdName) From a68068813ab3a9f20ceeebef5031ddc0a708589c Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Fri, 29 Oct 2021 18:43:18 +0200 Subject: [PATCH 112/344] Hugepages TC fix (#410) The hugepages TC implementation had two main problems: - The /proc/meminfo file shows only the total hugepages for the default size, so in case hugepages count of a different size were configured in the machineconfig, that file won't reflect that count, so it caused the TC to fail. - When MC has hugepages config per numa, this config is set in systemd.units, not in the kernelArguments. With this change, the actual node hugepages values are retrieved by reading these node files with "oc debug": /sys/devices/system/node/node/hugepages/hugepages--kB/nr_hugepages E.g: /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages In case machineconfig has valid hugepages config in the systemd.units (one unit per numa and size), this info will be compared against node values. Otherwise, kernelArguments will be parsed and compared against the total count of hugepages of that size found in the node. This is because, usually, when no hugepages per numa were explicitly set in the MC, the value is balanced between all the numas in the system. In other words, if system has 2 numas and kernelArguments show 256 hugepages of 2M size, each numa will have 128 hugepages. --- test-network-function/platform/suite.go | 404 ++++++++++++++++++++++-- 1 file changed, 381 insertions(+), 23 deletions(-) diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index f18e06919..fd3cfc53a 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -19,9 +19,12 @@ package platform import ( "encoding/json" "fmt" + "path" "regexp" + "sort" "strconv" "strings" + "time" log "github.com/sirupsen/logrus" @@ -38,9 +41,8 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/cnffsdiff" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/containerid" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/currentkernelcmdlineargs" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/hugepages" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/mckernelarguments" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodehugepages" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodemcname" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodetainted" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/podnodename" @@ -52,6 +54,67 @@ import ( "github.com/test-network-function/test-network-function/test-network-function/results" ) +const ( + RhelDefaultHugepagesz = 2048 // kB + RhelDefaultHugepages = 0 + HugepagesParam = "hugepages" + HugepageszParam = "hugepagesz" + DefaultHugepagesz = "default_hugepagesz" + KernArgsKeyValueSplitLen = 2 +) + +var ( + commandHandlerFilePath = path.Join(common.PathRelativeToRoot, "pkg", "tnf", "handlers", "command", "command.json") + mcGetterCommandTimeout = time.Second * 30 +) + +type hugePagesConfig struct { + hugepagesSize int // size in kb + hugepagesCount int +} + +// numaHugePagesPerSize maps a numa id to an array of hugePagesConfig structs. +type numaHugePagesPerSize map[int][]hugePagesConfig + +// String is the stringer implementation for the numaHugePagesPerSize type so debug/info +// lines look better. +func (numaHugepages numaHugePagesPerSize) String() string { + // Order numa ids/indexes + numaIndexes := make([]int, 0) + for numaIdx := range numaHugepages { + numaIndexes = append(numaIndexes, numaIdx) + } + sort.Ints(numaIndexes) + + str := "" + for _, numaIdx := range numaIndexes { + hugepagesPerSize := numaHugepages[numaIdx] + str += fmt.Sprintf("Numa=%d ", numaIdx) + for _, hugepages := range hugepagesPerSize { + str += fmt.Sprintf("[Size=%dkB Count=%d] ", hugepages.hugepagesSize, hugepages.hugepagesCount) + } + } + return str +} + +// machineConfig maps a json machineconfig object to get the KernelArguments and systemd units info. +type machineConfig struct { + Spec struct { + KernelArguments []string `json:"kernelArguments"` + Config struct { + Systemd struct { + Units []systemdHugePagesUnit `json:"units"` + } + } `json:"config"` + } `json:"spec"` +} + +// systemdHugePagesUnit maps a systemd unit in a machineconfig json object. +type systemdHugePagesUnit struct { + Contents string `json:"contents"` + Name string `json:"name"` +} + // // All actual test code belongs below here. Utilities belong above. // @@ -357,39 +420,334 @@ func testTainted(env *config.TestEnvironment) { }) } -func getNodeMcHugepages(nodeName string) (hugePagesCount, hugePagesSize int) { - context := common.GetContext() - mcName := getMcName(context, nodeName) - hugepageTester := hugepages.NewHugepages(common.DefaultTimeout, mcName) - test, err := tnf.NewTest(context.GetExpecter(), hugepageTester, []reel.Handler{hugepageTester}, context.GetErrorChannel()) +func runAndValidateCommand(command string, context *interactive.Context, failureCallbackFun func()) (match string) { + log.Debugf("Launching generic command handler for cmd: %s", command) + + values := make(map[string]interface{}) + values["COMMAND"] = command + values["TIMEOUT"] = mcGetterCommandTimeout.Nanoseconds() + + tester, handlers, result, err := generic.NewGenericFromMap(commandHandlerFilePath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) - testResult, err := test.Run() - gomega.Expect(testResult).To(gomega.Equal(tnf.SUCCESS)) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) + gomega.Expect(test).ToNot(gomega.BeNil()) gomega.Expect(err).To(gomega.BeNil()) - return hugepageTester.GetHugepages(), hugepageTester.GetHugepagesz() + test.RunAndValidateWithFailureCallback(failureCallbackFun) + + genericTest := (*tester).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + return genericTest.GetMatches()[0].Match +} + +func hugepageSizeToInt(s string) int { + num, _ := strconv.Atoi(s[:len(s)-1]) + unit := s[len(s)-1] + switch unit { + case 'M': + num *= 1024 + case 'G': + num *= 1024 * 1024 + } + + return num +} + +func logMcKernelArgumentsHugepages(hugepagesPerSize map[int]int, defhugepagesz int) { + logStr := fmt.Sprintf("MC KernelArguments hugepages config: default_hugepagesz=%d-kB", defhugepagesz) + for size, count := range hugepagesPerSize { + logStr += fmt.Sprintf(", size=%dkB - count=%d", size, count) + } + log.Info(logStr) +} + +// getMcHugepagesFromMcKernelArguments gets the hugepages params from machineconfig's kernelArguments +func getMcHugepagesFromMcKernelArguments(mc *machineConfig) (hugepagesPerSize map[int]int, defhugepagesz int, err error) { + defhugepagesz = RhelDefaultHugepagesz + hugepagesPerSize = map[int]int{} + + hugepagesz := 0 + for _, arg := range mc.Spec.KernelArguments { + keyValueSlice := strings.Split(arg, "=") + if len(keyValueSlice) != KernArgsKeyValueSplitLen { + // Some kernel arguments don't come in name=value + continue + } + + key, value := keyValueSlice[0], keyValueSlice[1] + if key == HugepagesParam && value != "" { + if _, sizeFound := hugepagesPerSize[hugepagesz]; !sizeFound { + return map[int]int{}, RhelDefaultHugepagesz, fmt.Errorf("found hugepages count without size in kernelArguments: %v", mc.Spec.KernelArguments) + } + hugepages, _ := strconv.Atoi(value) + hugepagesPerSize[hugepagesz] = hugepages + } + + if key == HugepageszParam && value != "" { + hugepagesz = hugepageSizeToInt(value) + // Create new map entry for this size + hugepagesPerSize[hugepagesz] = 0 + } + + if key == DefaultHugepagesz && value != "" { + defhugepagesz = hugepageSizeToInt(value) + // In case only default_hugepagesz and hugepages values are provided. The actual value should be + // parsed next and this default value overwritten. + hugepagesPerSize[defhugepagesz] = RhelDefaultHugepages + hugepagesz = defhugepagesz + } + } + + if len(hugepagesPerSize) == 0 { + hugepagesPerSize[RhelDefaultHugepagesz] = RhelDefaultHugepages + log.Warnf("No hugepages size found in node's machineconfig. Defaulting to size=%dkB (count=%d)", RhelDefaultHugepagesz, RhelDefaultHugepages) + } + + logMcKernelArgumentsHugepages(hugepagesPerSize, defhugepagesz) + return hugepagesPerSize, defhugepagesz, nil +} + +// getNodeNumaHugePages gets the actual node's hugepages config based on /sys/devices/system/node/nodeX files. +func getNodeNumaHugePages(node *config.NodeConfig) (hugepages numaHugePagesPerSize, err error) { + const cmd = "for file in `find /sys/devices/system/node/ -name nr_hugepages`; do echo $file count:`cat $file` ; done" + const outputRegex = `node(\d+).*hugepages-(\d+)kB.* count:(\d+)` + const numRegexFields = 4 + + // This command must run inside the node, so we'll need the node's context to run commands inside the debug daemonset pod. + context := interactive.NewContext(node.Oc.GetExpecter(), node.Oc.GetErrorChannel()) + var commandErr error + hugepagesCmdOut := runAndValidateCommand(cmd, context, func() { + commandErr = fmt.Errorf("failed to get node %s hugepages per numa", node.Name) + }) + if commandErr != nil { + return numaHugePagesPerSize{}, commandErr + } + + hugepages = numaHugePagesPerSize{} + r := regexp.MustCompile(outputRegex) + for _, line := range strings.Split(hugepagesCmdOut, "\n") { + values := r.FindStringSubmatch(line) + if len(values) != numRegexFields { + return numaHugePagesPerSize{}, fmt.Errorf("failed to parse node's numa hugepages output line:%s", line) + } + + numaNode, _ := strconv.Atoi(values[1]) + hpSize, _ := strconv.Atoi(values[2]) + hpCount, _ := strconv.Atoi(values[3]) + + hugepagesCfg := hugePagesConfig{ + hugepagesCount: hpCount, + hugepagesSize: hpSize, + } + + if numaHugepagesCfg, exists := hugepages[numaNode]; exists { + numaHugepagesCfg = append(numaHugepagesCfg, hugepagesCfg) + hugepages[numaNode] = numaHugepagesCfg + } else { + hugepages[numaNode] = []hugePagesConfig{hugepagesCfg} + } + } + + log.Infof("Node %s hugepages: %s", node.Name, hugepages) + return hugepages, nil +} + +// getMachineConfig gets the machineconfig in json format does the unmarshalling. +func getMachineConfig(mcName string) (machineConfig, error) { + var commandErr error + + // Local shell context is needed for the command handler. + context := common.GetContext() + mcJSON := runAndValidateCommand(fmt.Sprintf("oc get mc %s -o json", mcName), context, func() { + commandErr = fmt.Errorf("failed to get json machineconfig %s", mcName) + }) + if commandErr != nil { + return machineConfig{}, commandErr + } + + var mc machineConfig + err := json.Unmarshal([]byte(mcJSON), &mc) + if err != nil { + return machineConfig{}, fmt.Errorf("failed to unmarshall (err: %v)", err) + } + + return mc, nil +} + +// getMcSystemdUnitsHugepagesConfig gets the hugepages information from machineconfig's systemd units. +func getMcSystemdUnitsHugepagesConfig(mc *machineConfig) (hugepages numaHugePagesPerSize, err error) { + const UnitContentsRegexMatchLen = 4 + hugepages = numaHugePagesPerSize{} + + r := regexp.MustCompile(`(?ms)HUGEPAGES_COUNT=(\d+).*HUGEPAGES_SIZE=(\d+).*NUMA_NODE=(\d+)`) + for _, unit := range mc.Spec.Config.Systemd.Units { + unit.Name = strings.Trim(unit.Name, "\"") + if !strings.Contains(unit.Name, "hugepages-allocation") { + continue + } + unit.Contents = strings.Trim(unit.Contents, "\"") + values := r.FindStringSubmatch(unit.Contents) + if len(values) < UnitContentsRegexMatchLen { + return numaHugePagesPerSize{}, fmt.Errorf("unable to get hugepages values from mc (contents=%s)", unit.Contents) + } + + numaNode, _ := strconv.Atoi(values[3]) + hpSize, _ := strconv.Atoi(values[2]) + hpCount, _ := strconv.Atoi(values[1]) + + hugepagesCfg := hugePagesConfig{ + hugepagesCount: hpCount, + hugepagesSize: hpSize, + } + + if numaHugepagesCfg, exists := hugepages[numaNode]; exists { + numaHugepagesCfg = append(numaHugepagesCfg, hugepagesCfg) + hugepages[numaNode] = numaHugepagesCfg + } else { + hugepages[numaNode] = []hugePagesConfig{hugepagesCfg} + } + } + + if len(hugepages) > 0 { + log.Infof("Machineconfig's systemd.units hugepages: %v", hugepages) + } else { + log.Infof("No hugepages found in machineconfig system.units") + } + + return hugepages, nil +} + +// testNodeHugepagesWithMcSystemd compares the node's hugepages values against the mc's systemd units ones. +func testNodeHugepagesWithMcSystemd(nodeName string, nodeNumaHugePages, mcSystemdHugepages numaHugePagesPerSize) (bool, error) { + // Iterate through mc's numas and make sure they exist and have the same sizes and values in the node. + for mcNumaIdx, mcNumaHugepageCfgs := range mcSystemdHugepages { + nodeNumaHugepageCfgs, exists := nodeNumaHugePages[mcNumaIdx] + if !exists { + return false, fmt.Errorf("node %s has no hugepages config for machine config's numa %d", nodeName, mcNumaIdx) + } + + // For this numa, iterate through each of the mc's hugepages sizes and compare with node ones. + for _, mcHugepagesCfg := range mcNumaHugepageCfgs { + configMatching := false + for _, nodeHugepagesCfg := range nodeNumaHugepageCfgs { + if nodeHugepagesCfg.hugepagesSize == mcHugepagesCfg.hugepagesSize && nodeHugepagesCfg.hugepagesCount == mcHugepagesCfg.hugepagesCount { + log.Infof("MC numa=%d, hugepages count:%d, size:%d match node ones: %s", + mcNumaIdx, mcHugepagesCfg.hugepagesCount, mcHugepagesCfg.hugepagesSize, nodeNumaHugePages) + configMatching = true + break + } + } + if !configMatching { + return false, fmt.Errorf(fmt.Sprintf("MC numa=%d, hugepages (count:%d, size:%d) not matching node ones: %s", + mcNumaIdx, mcHugepagesCfg.hugepagesCount, mcHugepagesCfg.hugepagesSize, nodeNumaHugePages)) + } + } + } + + return true, nil +} + +// testNodeHugepagesWithKernelArgs compares node hugepages against kernelArguments config. +// The total count of hugepages of the size defined in the kernelArguments must match the kernArgs' hugepages value. +// For other sizes, the sum should be 0. +func testNodeHugepagesWithKernelArgs(nodeName string, nodeNumaHugePages numaHugePagesPerSize, kernelArgsHugepagesPerSize map[int]int) (bool, error) { + for size, count := range kernelArgsHugepagesPerSize { + total := 0 + for numaIdx, numaHugepages := range nodeNumaHugePages { + found := false + for _, hugepages := range numaHugepages { + if hugepages.hugepagesSize == size { + total += hugepages.hugepagesCount + found = true + break + } + } + if !found { + return false, fmt.Errorf("node %s: numa %d has no hugepages of size %d", nodeName, numaIdx, size) + } + } + + if total == count { + log.Infof("kernelArguments' hugepages count:%d, size:%d match total node ones for that size.", count, size) + } else { + return false, fmt.Errorf("node %s: total hugepages of size %d won't match (node count=%d, expected=%d)", + nodeName, size, total, count) + } + } + + return true, nil +} + +func getNodeMachineConfig(nodeName string, machineconfigs map[string]machineConfig) machineConfig { + mcName := strings.Trim(getMcName(common.GetContext(), nodeName), "\"") + log.Infof("Node %s is using machineconfig %s", nodeName, mcName) + + if mc, exists := machineconfigs[mcName]; exists { + log.Infof("MC %s: json already parsed.", mcName) + return mc + } + + mc, err := getMachineConfig(mcName) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Unable to unmarshall mc %s from node %s", mcName, nodeName)) + } + machineconfigs[mcName] = mc + + return mc } func testHugepages(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestHugepagesNotManuallyManipulated) ginkgo.It(testID, func() { + // Map to save already retrieved and parsed machineconfigs. + machineconfigs := map[string]machineConfig{} var badNodes []string + for _, node := range env.NodesUnderTest { if !node.IsWorker() || !node.HasDebugPod() { continue } - nodeOc := node.Oc - ginkgo.By("Should return machineconfig hugepages configuration of node " + node.Name) - nodeHugePagesCount, nodeHugePagesSize := getNodeMcHugepages(node.Name) - - ginkgo.By(fmt.Sprintf("Node's machine config hugepages=%d/hugepagesz=%d values should match the actual ones in the node.", - nodeHugePagesCount, nodeHugePagesSize)) - tester := nodehugepages.NewNodeHugepages(common.DefaultTimeout, nodeHugePagesSize, nodeHugePagesCount) - test, err := tnf.NewTest(nodeOc.GetExpecter(), tester, []reel.Handler{tester}, nodeOc.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - test.RunWithFailureCallback(func() { - badNodes = append(badNodes, node.Name) - ginkgo.By(fmt.Sprintf("Node=%s hugepage config does not match machineconfig", node.Name)) - }) + + ginkgo.By(fmt.Sprintf("Should get node %s numa's hugepages values.", node.Name)) + nodeNumaHugePages, err := getNodeNumaHugePages(node) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Unable to get node hugepages values from node %s", node.Name)) + } + + // Get and parse node's machineconfig, in case it's not already parsed. + mc := getNodeMachineConfig(node.Name, machineconfigs) + + ginkgo.By("Should parse machineconfig's kernelArguments and systemd's hugepages units.") + mcSystemdHugepages, err := getMcSystemdUnitsHugepagesConfig(&mc) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Failed to get MC systemd hugepages config. Error: %v", err)) + } + + // KernelArguments params will only be used in case no systemd units were found. + if len(mcSystemdHugepages) == 0 { + ginkgo.By("Comparing MC KernelArguments hugepages info against node values.") + hugepagesPerSize, _, err := getMcHugepagesFromMcKernelArguments(&mc) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Unable to get mc kernelArguments hugepages from node %s. Error: %v", node.Name, err)) + } + if pass, err := testNodeHugepagesWithKernelArgs(node.Name, nodeNumaHugePages, hugepagesPerSize); !pass { + log.Error(err) + badNodes = append(badNodes, node.Name) + } + } else { + ginkgo.By("Comparing MC Systemd hugepages info against node values.") + if pass, err := testNodeHugepagesWithMcSystemd(node.Name, nodeNumaHugePages, mcSystemdHugepages); !pass { + log.Error(err) + badNodes = append(badNodes, node.Name) + } + } } gomega.Expect(badNodes).To(gomega.BeNil()) }) From 3a446be2df03119c70a548cdfa06b747176f59c8 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 29 Oct 2021 13:26:16 -0500 Subject: [PATCH 113/344] Enabling intrusive tests in Github workflow (#423) --- .github/workflows/pre-main.yaml | 2 +- .github/workflows/tnf-image.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 5dbe4217f..10e95da02 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -13,7 +13,7 @@ env: IMAGE_TAG: unstable TNF_CONTAINER_CLIENT: docker TNF_MINIKUBE_ONLY: true - TNF_NON_INTRUSIVE_ONLY: true + TNF_NON_INTRUSIVE_ONLY: false TNF_DISABLE_CONFIG_AUTODISCOVER: false TNF_CONFIG_DIR: /tmp/tnf/config TNF_OUTPUT_DIR: /tmp/tnf/output diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 7b1fa4209..7b5da5d65 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -22,7 +22,7 @@ env: IMAGE_TAG: latest TNF_CONTAINER_CLIENT: docker TNF_MINIKUBE_ONLY: true - TNF_NON_INTRUSIVE_ONLY: true + TNF_NON_INTRUSIVE_ONLY: false TNF_DISABLE_CONFIG_AUTODISCOVER: false TNF_CONFIG_DIR: /tmp/tnf/config TNF_OUTPUT_DIR: /tmp/tnf/output From f4677488683f659210f00b558a371d25f9341a4d Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 29 Oct 2021 14:52:06 -0500 Subject: [PATCH 114/344] Fix for Ginkgo v2 deprecation warnings and filtering bug (#420) * Fix for Ginkgo v2 deprecation warnings and filtering bug * lint * PR Review and bugfix --- DEVELOPING.md | 26 ++++++++++++++---------- pkg/tnf/test.go | 15 -------------- run-cnf-suites.sh | 6 +++--- test-network-function/lifecycle/suite.go | 23 ++++++++++++++------- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/DEVELOPING.md b/DEVELOPING.md index ef5a74336..b9b5a1f4e 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -627,30 +627,34 @@ The command relays on golang templates located in [pkg/tnf/handlers/handler_temp The result of each test execution is included in the claim file. Sometimes it is convenient to add informational messages regarding the test execution. -For this purpose we have an additional section in the claim file (see `suite.extraInfoKey` in suite_test.go). -In order to add informational messages to your test use the function `tnf.CreateTestExtraInfoWriter`. -This function adds an entry for your test in the claim file. -The return value is a function. -The returned function can be called to add a message to the test entry. +In order to add informational messages to your test use the function `ginkgo.GinkgoWriter`. +This function adds an additional message that will appear in the `CapturedTestOutput` section of the claim file, together witht the output of the by directives. Each added message will be written to claim file even if test failed or error occurred in the middle of the test. Example usage: ```go ginkgo.It("Should do what I tell it to do", func(){ - // do some work - // create test info writer - myWriter := tnf.CreateTestExtraInfoWriter() // do some more work // add info - myWriter("important info part 1") + _, err := ginkgo.GinkgoWriter.Write([]byte("important info part 1")) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } + // more work // more info - myWriter("important info part 2") + _, err := ginkgo.GinkgoWriter.Write([]byte("important info part 2")) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } // error if err != nil { return } // last info - myWriter("important info part last") + _, err := ginkgo.GinkgoWriter.Write([]byte("important info part last")) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } }) ``` \ No newline at end of file diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index 843449e41..c73d22514 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -22,7 +22,6 @@ import ( log "github.com/sirupsen/logrus" expect "github.com/google/goexpect" - "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" @@ -41,20 +40,6 @@ const ( // use WriteTestExtraInfo for writing to it var TestsExtraInfo = []map[string][]string{} -// CreateTestExtraInfoWriter creates a function that writes info messages for a specific test -// info messages that were already added by calling the function will exist in the claim file -func CreateTestExtraInfoWriter() func(string) { - testName := ginkgo.CurrentGinkgoTestDescription().FullTestText - if testName == "" { - return func(string) {} - } - extraInfo := map[string][]string{testName: nil} - TestsExtraInfo = append(TestsExtraInfo, extraInfo) - return func(info string) { - extraInfo[testName] = append(extraInfo[testName], info) - } -} - // ExitCodeMap maps a test result value to a more appropriate Unix return code. var ExitCodeMap = map[int]int{ SUCCESS: 0, diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index 40e5bb0ca..b3457ce9e 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -45,7 +45,7 @@ while [[ $1 == -* ]]; do shift done # specify Junit report file name. -GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC -ginkgo.reportFile $OUTPUT_LOC/cnf-certification-tests_junit.xml -ginkgo.v -test.v" +GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC --ginkgo.junit-report $OUTPUT_LOC/cnf-certification-tests_junit.xml -ginkgo.v -test.v" # If no focus is set then display usage and quit with a non-zero exit code. @@ -73,7 +73,7 @@ echo "Running with skip '$SKIP'" echo "Report will be output to '$OUTPUT_LOC'" echo "ginkgo arguments '${GINKGO_ARGS}'" SKIP_STRING="" -if [ -z $SKIP ]; then - $SKIP_STRING=-ginkgo.skip="$SKIP" +if [ -n "$SKIP" ]; then + SKIP_STRING=-ginkgo.skip="$SKIP" fi cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" $SKIP_STRING ${GINKGO_ARGS} diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 11a75718e..8679d1b9d 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -212,14 +212,16 @@ func testNodeSelector(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing pod nodeSelector %s/%s", podNamespace, podName)) - infoWriter := tnf.CreateTestExtraInfoWriter() tester := nodeselector.NewNodeSelector(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) test.RunAndValidateWithFailureCallback(func() { msg := fmt.Sprintf("The pod specifies nodeSelector/nodeAffinity field, you might want to change it, %s %s", podNamespace, podName) log.Warn(msg) - infoWriter(msg) + _, err := ginkgo.GinkgoWriter.Write([]byte(msg)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } }) } }) @@ -234,7 +236,6 @@ func testGracePeriod(env *config.TestEnvironment) { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace ginkgo.By(fmt.Sprintf("Testing pod terminationGracePeriod %s %s", podNamespace, podName)) - infoWriter := tnf.CreateTestExtraInfoWriter() tester := graceperiod.NewGracePeriod(common.DefaultTimeout, podName, podNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -243,7 +244,10 @@ func testGracePeriod(env *config.TestEnvironment) { if gracePeriod == defaultTerminationGracePeriod { msg := fmt.Sprintf("%s %s has terminationGracePeriod set to %d, you might want to change it", podNamespace, podName, defaultTerminationGracePeriod) log.Warn(msg) - infoWriter(msg) + _, err := ginkgo.GinkgoWriter.Write([]byte(msg)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } } } }) @@ -392,7 +396,6 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { values := make(map[string]interface{}) values["DEPLOYMENT_NAME"] = deployment values["DEPLOYMENT_NAMESPACE"] = podNamespace - infoWriter := tnf.CreateTestExtraInfoWriter() tester, handlers, result, err := generic.NewGenericFromMap(relativePodTestPath, common.RelativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(result).ToNot(gomega.BeNil()) @@ -409,13 +412,19 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { msg := fmt.Sprintf("The deployment replica count is %d, but a podAntiAffinity rule is not defined, "+ "you might want to change it in deployment %s in namespace %s", replica, deployment, podNamespace) log.Warn(msg) - infoWriter(msg) + _, err := ginkgo.GinkgoWriter.Write([]byte(msg)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } } else { msg := fmt.Sprintf("The deployment replica count is %d. Pod replica should be > 1 with an "+ "podAntiAffinity rule defined . You might want to change it in deployment %s in namespace %s", replica, deployment, podNamespace) log.Warn(msg) - infoWriter(msg) + _, err := ginkgo.GinkgoWriter.Write([]byte(msg)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } } }) } From bac107a7c57a3fb623f8833c97f9df577e5d0c78 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 29 Oct 2021 16:28:54 -0400 Subject: [PATCH 115/344] small fixes to diagnostic suite (#422) --- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 2 +- pkg/tnf/handlers/nodedebug/nodedebug.go | 2 +- test-network-function/common/suite.go | 7 +++-- test-network-function/diagnostic/suite.go | 30 ++++++++++++-------- test-network-function/observability/suite.go | 13 +++++---- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index 0c2652351..a772292c8 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -96,7 +96,7 @@ func (p *CnfFsDiff) ReelEOF() { // Command returns command line args for checking the fs difference between a container and it's image func Command(containerID string) []string { - return []string{"chroot", "/host", "podman", "diff", "--format", "json", containerID} + return []string{"/host/bin/podman", "diff", "--format", "json", containerID} } // NewFsDiff creates a new `FsDiff` test which checks the fs difference between a container and it's image diff --git a/pkg/tnf/handlers/nodedebug/nodedebug.go b/pkg/tnf/handlers/nodedebug/nodedebug.go index 9ea8fc8c7..cf0e6271c 100644 --- a/pkg/tnf/handlers/nodedebug/nodedebug.go +++ b/pkg/tnf/handlers/nodedebug/nodedebug.go @@ -50,7 +50,7 @@ func NewNodeDebug(timeout time.Duration, nodeName, command string, trim, split b timeout: timeout, result: tnf.ERROR, args: []string{ - "chroot", "/host", command, + command, }, Trim: trim, Split: split, diff --git a/test-network-function/common/suite.go b/test-network-function/common/suite.go index 50a4f6ff6..fdbc4e545 100644 --- a/test-network-function/common/suite.go +++ b/test-network-function/common/suite.go @@ -29,14 +29,17 @@ var _ = ginkgo.BeforeSuite(func() { for name := range autodiscover.GetNodesList() { autodiscover.DeleteDebugLabel(name) } - env = configpkg.GetTestEnvironment() - env.LoadAndRefresh() }) var _ = ginkgo.AfterSuite(func() { // clean up added label to nodes log.Info("clean up added labels to nodes") + env = configpkg.GetTestEnvironment() + env.LoadAndRefresh() for name, node := range env.NodesUnderTest { + if !(node.HasDebugPod()) { + continue + } node.Oc.Close() autodiscover.DeleteDebugLabel(name) } diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index f7260a07d..5724880da 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -62,12 +62,14 @@ var ( // schemaPath is the path to the generic-test.schema.json JSON schema relative to the project root. schemaPath = path.Join("schemas", "generic-test.schema.json") + + // retrieve the singleton instance of test environment + env *config.TestEnvironment = config.GetTestEnvironment() ) var _ = ginkgo.Describe(common.DiagnosticTestKey, func() { conf, _ := ginkgo.GinkgoConfiguration() if testcases.IsInFocus(conf.FocusStrings, common.DiagnosticTestKey) { - env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) @@ -184,7 +186,7 @@ func getWorkerNodeName(env *config.TestEnvironment) string { } func listNodeCniPlugins(nodeName string) []CniPlugin { - const command = "jq -r .name,.cniVersion /etc/cni/net.d/*" + const command = "cat /host/etc/cni/net.d/* | chroot /host jq -r .name,.cniVersion" result := []CniPlugin{} nodes := config.GetTestEnvironment().NodesUnderTest context := nodes[nodeName].Oc @@ -216,7 +218,7 @@ func testCniPlugins() { ginkgo.Skip("can't use 'oc debug' in minikube") } // get name of a master node - env := config.GetTestEnvironment() + env = config.GetTestEnvironment() nodeName := getMasterNodeName(env) gomega.Expect(nodeName).ToNot(gomega.BeEmpty()) // get CNI plugins from node @@ -228,7 +230,7 @@ func testNodesHwInfo() { if common.IsMinikube() { ginkgo.Skip("can't use 'oc debug' in minikube") } - env := config.GetTestEnvironment() + env = config.GetTestEnvironment() masterNodeName := getMasterNodeName(env) gomega.Expect(masterNodeName).ToNot(gomega.BeEmpty()) workerNodeName := getWorkerNodeName(env) @@ -249,8 +251,9 @@ func getNodeLscpu(nodeName string) map[string]string { const command = "lscpu" const numSplitSubstrings = 2 result := map[string]string{} - env := config.GetTestEnvironment().NodesUnderTest - context := env[nodeName].Oc + env = config.GetTestEnvironment() + nodes := env.NodesUnderTest + context := nodes[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -266,8 +269,9 @@ func getNodeIfconfig(nodeName string) map[string][]string { const command = "ifconfig" const numSplitSubstrings = 2 result := map[string][]string{} - env := config.GetTestEnvironment().NodesUnderTest - context := env[nodeName].Oc + env = config.GetTestEnvironment() + nodes := env.NodesUnderTest + context := nodes[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -289,8 +293,9 @@ func getNodeIfconfig(nodeName string) map[string][]string { func getNodeLsblk(nodeName string) interface{} { const command = "lsblk -J" - env := config.GetTestEnvironment().NodesUnderTest - context := env[nodeName].Oc + env = config.GetTestEnvironment() + nodes := env.NodesUnderTest + context := nodes[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, false, false) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) @@ -303,8 +308,9 @@ func getNodeLsblk(nodeName string) interface{} { func getNodeLspci(nodeName string) []string { const command = "lspci" - env := config.GetTestEnvironment().NodesUnderTest - context := env[nodeName].Oc + env = config.GetTestEnvironment() + nodes := env.NodesUnderTest + context := nodes[nodeName].Oc tester := nodedebug.NewNodeDebug(defaultTestTimeout, nodeName, command, true, true) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 969bee18f..528fc94d1 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -48,24 +48,25 @@ var ( relativeCrdTestPath = path.Join(common.PathRelativeToRoot, crdTestPath) // testCrdsTimeout is the timeout in seconds for the CRDs TC. testCrdsTimeout = 10 * time.Second + // retrieve the singleton instance of test environment + env *config.TestEnvironment = config.GetTestEnvironment() ) - var _ = ginkgo.Describe(common.ObservabilityTestKey, func() { conf, _ := ginkgo.GinkgoConfiguration() + if testcases.IsInFocus(conf.FocusStrings, common.ObservabilityTestKey) { - env := config.GetTestEnvironment() ginkgo.BeforeEach(func() { env.LoadAndRefresh() gomega.Expect(len(env.PodsUnderTest)).ToNot(gomega.Equal(0)) gomega.Expect(len(env.ContainersUnderTest)).ToNot(gomega.Equal(0)) }) ginkgo.ReportAfterEach(results.RecordResult) - testLogging(env) - testCrds(env) + testLogging() + testCrds() } }) -func testLogging(env *config.TestEnvironment) { +func testLogging() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestLoggingIdentifier) ginkgo.It(testID, func() { for _, cut := range env.ContainersUnderTest { @@ -95,7 +96,7 @@ func loggingTest(c configsections.ContainerIdentifier) { test.RunAndValidate() } -func testCrds(env *config.TestEnvironment) { +func testCrds() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestCrdsStatusSubresourceIdentifier) ginkgo.It(testID, func() { ginkgo.By("CRDs should have a status subresource") From e532df8e46ebe25dc45aad8defdaf8905c648eba Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 1 Nov 2021 16:23:52 -0500 Subject: [PATCH 116/344] Fix for duplicate dangling sessions un-expectedly closing (#424) Fix for duplicate dangling sessions un-expectedly closing --- pkg/config/config.go | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0b1bc9c78..e5407a2b5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -199,25 +199,34 @@ func (env *TestEnvironment) LoadAndRefresh() { } env.doAutodiscover() } else if env.needsRefresh { - log.Debug("clean up environment Test structure") - env.Config.Partner = configsections.TestPartner{} - env.Config.TestTarget = configsections.TestTarget{} - env.TestOrchestrator = nil - for name, node := range env.NodesUnderTest { - if node.HasDebugPod() { - node.Oc.Close() - autodiscover.DeleteDebugLabel(name) - } - } - env.NodesUnderTest = nil - env.Config.Nodes = nil - env.DebugContainers = nil - log.Debug("start auto discovery") + env.reset() env.doAutodiscover() } } +func (env *TestEnvironment) reset() { + log.Debug("clean up environment Test structure") + env.Config.Partner = configsections.TestPartner{} + env.Config.TestTarget = configsections.TestTarget{} + env.TestOrchestrator = nil + // Delete Oc debug sessions before re-creating them + for name, node := range env.NodesUnderTest { + if node.HasDebugPod() { + node.Oc.Close() + autodiscover.DeleteDebugLabel(name) + } + } + // Delete all remaining sessions before re-creating them + for _, cut := range env.ContainersUnderTest { + cut.Oc.Close() + } + env.NodesUnderTest = nil + env.Config.Nodes = nil + env.DebugContainers = nil +} + func (env *TestEnvironment) doAutodiscover() { + log.Debug("start auto discovery") if len(env.Config.TargetNameSpaces) != 1 { log.Fatal("a single namespace should be specified in config file") } From 283ed35c87a15bb606c3aaadd5fab8bc03736997 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 1 Nov 2021 18:00:45 -0500 Subject: [PATCH 117/344] test fix (#427) Co-authored-by: Salaheddine Hamadi --- test-network-function/lifecycle/suite.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 8679d1b9d..78f249bf8 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -313,6 +313,9 @@ func testPodsRecreation(env *config.TestEnvironment) { closeOcSessionsByNode(env.PartnerContainers, n.Name) // drain node drainNode(n.Name) // should go in this + + waitForAllDeploymentsReady(env.NameSpaceUnderTest, scalingTimeout, scalingPollingPeriod) + // verify deployments are ready again _, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) if len(notReadyDeployments) != 0 { From 91009fef0e08085835f2cd1ce14f4a2586748dfd Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 1 Nov 2021 18:15:57 -0500 Subject: [PATCH 118/344] Checks if the claim/junit directory exists (#426) Co-authored-by: Salaheddine Hamadi --- README.md | 2 +- pkg/utils/utils.go | 20 +++++++++++++++++++- test-network-function/suite_test.go | 6 ++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 890512122..d5a169a01 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ It's recommended to clean up disk space and make sure there's enough resources t There are several required arguments: * `-t` gives the local directory that contains tnf config files set up for the test. -* `-o` gives the local directory that the test results will be available in once the container exits. +* `-o` gives the local directory that the test results will be available in once the container exits. This directory must exist in order for the claim file to be written. * `-f` gives the list of suites to be run, space separated. Optional arguments are: diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 9d7ff8385..9ce912021 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,6 +1,13 @@ package utils -import "strings" +import ( + "errors" + "os" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" +) // ArgListToMap takes a list of strings of the form "key=value" and translate it into a map // of the form {key: value} @@ -27,3 +34,14 @@ func FilterArray(vs []string, f func(string) bool) []string { } return vsf } + +func CheckFileExists(path, name string) { + fullPath, _ := filepath.Abs(path) + if _, err := os.Stat(path); err == nil { + log.Infof("Path to %s file found and valid: %s ", name, fullPath) + } else if errors.Is(err, os.ErrNotExist) { + log.Fatalf("Path to %s file not found: %s , Exiting", name, fullPath) + } else { + log.Infof("Path to %s file not valid: %s , err=%s, exiting", name, fullPath, err) + } +} diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index edb047e91..81b467fb7 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -35,6 +35,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf" tnfcommon "github.com/test-network-function/test-network-function/pkg/tnf/handlers/common" + utils "github.com/test-network-function/test-network-function/pkg/utils" _ "github.com/test-network-function/test-network-function/test-network-function/accesscontrol" _ "github.com/test-network-function/test-network-function/test-network-function/certification" "github.com/test-network-function/test-network-function/test-network-function/common" @@ -120,6 +121,11 @@ func loadJUnitXMLIntoMap(result map[string]interface{}, junitFilename, key strin func TestTest(t *testing.T) { // set up input flags and register failure handlers. flag.Parse() + + // Checking if output directories exist + utils.CheckFileExists(*claimPath, "claim") + utils.CheckFileExists(*junitPath, "junit") + gomega.RegisterFailHandler(ginkgo.Fail) common.SetLogFormat() common.SetLogLevel() From cb1121570579671a87e67425735ec7ed634a4bfe Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Mon, 1 Nov 2021 19:30:02 -0400 Subject: [PATCH 119/344] Fix gitCommit static var (#425) * Fix gitCommit static var * Update run-container.sh Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- Makefile | 2 +- script/run-container.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0164fec1a..40f9c3eeb 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ build-cnf-tests: make build-catalog-md build-cnf-tests-debug: - PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_RELEASE} -X github.com/test-network-function/test-network-function/test-network-function.GitPreviousRelease=${GIT_PREVIOUS_RELEASE} -extldflags '-z relro -z now'" ./test-network-function + PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -X github.com/test-network-function/test-network-function/test-network-function.GitRelease=${GIT_RELEASE} -X github.com/test-network-function/test-network-function/test-network-function.GitPreviousRelease=${GIT_PREVIOUS_RELEASE} -extldflags '-z relro -z now'" ./test-network-function make build-catalog-md # run all CNF tests diff --git a/script/run-container.sh b/script/run-container.sh index fde51731d..df69f7cba 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -109,6 +109,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ -e TNF_PARTNER_NAMESPACE=$TNF_PARTNER_NAMESPACE \ -e TNF_PARTNER_REPO=$TNF_PARTNER_REPO \ + -e TNF_DEPLOYMENT_TIMEOUT=$TNF_DEPLOYMENT_TIMEOUT \ -e TNF_OC_DEBUG_IMAGE_ID=$TNF_OC_DEBUG_IMAGE_ID \ -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ From 91eee9a3efd1f9612104a7b450ec37a3512d4f7b Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Tue, 2 Nov 2021 15:07:31 +0200 Subject: [PATCH 120/344] Add SCC capability checks for IPC_LOCK and NET_RAW (#413) * Add SCC capability checks for IPC_LOCK and NET_RAW * fixed make lint * Updated the file privilegedpod.yml * fixed make lint * Resolved comment of Jun Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen --- CATALOG.md | 2 +- pkg/tnf/testcases/files/cnf/privilegedpod.yml | 2 ++ test-network-function/identifiers/identifiers.go | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CATALOG.md b/CATALOG.md index abed97e1a..7d9053935 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -20,7 +20,7 @@ Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/site Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/access-control/host-resource tests several aspects of CNF best practices, including: 1. The Pod does not have access to Host Node Networking. 2. The Pod does not have access to Host Node Ports. 3. The Pod cannot access Host Node IPC space. 4. The Pod cannot access Host Node PID space. 5. The Pod is not granted NET_ADMIN SCC. 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. +Description|http://test-network-function.com/testcases/access-control/host-resource tests several aspects of CNF best practices, including: 1. The Pod does not have access to Host Node Networking. 2. The Pod does not have access to Host Node Ports. 3. The Pod cannot access Host Node IPC space. 4. The Pod cannot access Host Node PID space. 5. The Pod is not granted NET_ADMIN SCC. 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. 9. The Pod is not granted NET_RAW SCC. 10. The Pod is not granted IPC_LOCK SCC. Result Type|normative Suggested Remediation|Ensure that each Pod in the CNF abides by the suggested best practices listed in the test description. In some rare cases, not all best practices can be followed. For example, some CNFs may be required to run as root. Such exceptions should be handled on a case-by-case basis, and should provide a proper justification as to why the best practice(s) cannot be followed. Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 diff --git a/pkg/tnf/testcases/files/cnf/privilegedpod.yml b/pkg/tnf/testcases/files/cnf/privilegedpod.yml index 91cac1843..56502649a 100644 --- a/pkg/tnf/testcases/files/cnf/privilegedpod.yml +++ b/pkg/tnf/testcases/files/cnf/privilegedpod.yml @@ -49,6 +49,8 @@ testcase: expectedstatus: - NET_ADMIN - SYS_ADMIN + - NET_RAW + - IPC_LOCK - name: ROOT_CHECK skiptest: true loop: 0 diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 63b6dfc60..69f02a2e5 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -263,6 +263,8 @@ cannot be followed.`, 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. +9. The Pod is not granted NET_RAW SCC. +10. The Pod is not granted IPC_LOCK SCC. `), BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, From 0da5bb36bd36f9346702a0e70c037d3ec8778782 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Tue, 2 Nov 2021 09:41:51 -0400 Subject: [PATCH 121/344] Add redhat_registry to enable offline run of daemon debug set (#429) Co-authored-by: Jun Chen --- script/run-container.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-container.sh b/script/run-container.sh index df69f7cba..b9bd155e7 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -111,6 +111,7 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ -e TNF_PARTNER_REPO=$TNF_PARTNER_REPO \ -e TNF_DEPLOYMENT_TIMEOUT=$TNF_DEPLOYMENT_TIMEOUT \ -e TNF_OC_DEBUG_IMAGE_ID=$TNF_OC_DEBUG_IMAGE_ID \ + -e REDHAT_RHEL_REGISTRY=$REDHAT_RHEL_REGISTRY \ -e LOG_LEVEL=$LOG_LEVEL \ -e PATH=/usr/bin:/usr/local/oc/bin \ $TNF_IMAGE \ From 4e5aec32b10b6356763303cae23c79b8933d598d Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Tue, 2 Nov 2021 16:19:46 +0100 Subject: [PATCH 122/344] Expecters Sent/Match debug output only when LOG_LEVEL=trace (#407) * Expecters Sent/Match debug output only when LOG_LEVEL=trace * Proposal to avoid circular dependencies. Not the most elegant way, though. Co-authored-by: Jun Chen --- pkg/config/autodiscover/autodiscover.go | 9 ++++++++- pkg/config/autodiscover/autodiscover_debug.go | 2 +- pkg/config/autodiscover/autodiscover_targets.go | 2 +- pkg/config/config.go | 10 +++++++++- pkg/tnf/interactive/shell.go | 4 ++-- test-network-function/common/env.go | 12 +++++++++++- test-network-function/suite_test.go | 3 +++ 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 7e6c38829..eb6ba7127 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -54,6 +54,8 @@ var ( schemaPath = path.Join("schemas", "generic-test.schema.json") // commandDriver stores the csi driver JSON output. commandDriver = make(map[string]interface{}) + + expectersVerboseModeEnabled = false ) // PerformAutoDiscovery checks the environment variable to see if autodiscovery should be performed @@ -97,7 +99,7 @@ func executeOcCommand(command string) (string, error) { values := make(map[string]interface{}) values["COMMAND"] = command values["TIMEOUT"] = ocCommandTimeOut.Nanoseconds() - context := interactive.GetContext() + context := interactive.GetContext(expectersVerboseModeEnabled) tester, handler, result, err := generic.NewGenericFromMap(pathToTestFile, relativeSchemaPath, values) gomega.Expect(err).To(gomega.BeNil()) @@ -183,3 +185,8 @@ func buildContainersFromPodResource(pr *PodResource) (containers []configsection } return } + +// EnableExpectersVerboseMode enables the verbose mode for expecters (Sent/Match output) +func EnableExpectersVerboseMode() { + expectersVerboseModeEnabled = true +} diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go index 3d6897230..82a999003 100644 --- a/pkg/config/autodiscover/autodiscover_debug.go +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -89,7 +89,7 @@ func CheckDebugDaemonset() { // checkDebugPodsReadiness helper function that returns true if the daemonset debug is deployed properly func checkDebugPodsReadiness() bool { - context := interactive.GetContext() + context := interactive.GetContext(expectersVerboseModeEnabled) tester := ds.NewDaemonSet(DefaultTimeout, debugDaemonSet, defaultNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) if err != nil { diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 2d879529e..891fb4d13 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -84,7 +84,7 @@ func FindTestTarget(labels []configsections.Label, target *configsections.TestTa func GetNodesList() (nodes map[string]configsections.Node) { nodes = make(map[string]configsections.Node) var nodeNames []string - context := interactive.GetContext() + context := interactive.GetContext(expectersVerboseModeEnabled) tester := nodenames.NewNodeNames(DefaultTimeout, map[string]*string{configsections.MasterLabel: nil}) test, _ := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) _, err := test.Run() diff --git a/pkg/config/config.go b/pkg/config/config.go index e5407a2b5..50180cce1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,6 +39,7 @@ const ( ) var ( + expectersVerboseModeEnabled = false // testEnvironment is the singleton instance of `TestEnvironment`, accessed through `GetTestEnvironment` testEnvironment TestEnvironment ) @@ -349,7 +350,7 @@ func (env *TestEnvironment) discoverNodes() { func (env *TestEnvironment) createContainers(containerDefinitions []configsections.ContainerConfig) map[configsections.ContainerIdentifier]*Container { createdContainers := make(map[configsections.ContainerIdentifier]*Container) for _, c := range containerDefinitions { - oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(true), interactive.SendTimeout(DefaultTimeout)) + oc := getOcSession(c.PodName, c.ContainerName, c.Namespace, DefaultTimeout, interactive.Verbose(expectersVerboseModeEnabled), interactive.SendTimeout(DefaultTimeout)) var defaultIPAddress = "UNKNOWN" var err error if _, ok := env.ContainersToExcludeFromConnectivityTests[c.ContainerIdentifier]; !ok { @@ -378,3 +379,10 @@ func (env *TestEnvironment) SetNeedsRefresh() { func GetTestEnvironment() *TestEnvironment { return &testEnvironment } + +// EnableExpectersVerboseMode enables the verbose mode for expecters (Sent/Match output) +func EnableExpectersVerboseMode() { + expectersVerboseModeEnabled = true + + autodiscover.EnableExpectersVerboseMode() +} diff --git a/pkg/tnf/interactive/shell.go b/pkg/tnf/interactive/shell.go index fe0afb18d..7bbf3ed25 100644 --- a/pkg/tnf/interactive/shell.go +++ b/pkg/tnf/interactive/shell.go @@ -39,8 +39,8 @@ func SpawnShell(spawner *Spawner, timeout time.Duration, opts ...Option) (*Conte // // // GetContext spawns a new shell session and returns its context -func GetContext() *Context { - context, err := SpawnShell(CreateGoExpectSpawner(), defaultTimeout, Verbose(true), SendTimeout(defaultTimeout)) +func GetContext(verbose bool) *Context { + context, err := SpawnShell(CreateGoExpectSpawner(), defaultTimeout, Verbose(verbose), SendTimeout(defaultTimeout)) if err != nil || context == nil || context.GetExpecter() == nil { log.Panicf("can't get a proper context for test execution") } diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index ee064da6e..2f1d0f6f8 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -43,9 +43,12 @@ var ( // DefaultTimeout for creating new interactive sessions (oc, ssh, tty) var DefaultTimeout = time.Duration(defaultTimeoutSeconds) * time.Second +// LogLevelTraceEnabled is saved to filter some debug trace logs (e.g. expecters Sent/Match) +var LogLevelTraceEnabled = false + // GetContext spawns a new shell session and returns its context func GetContext() *interactive.Context { - context, err := interactive.SpawnShell(interactive.CreateGoExpectSpawner(), DefaultTimeout, interactive.Verbose(true), interactive.SendTimeout(DefaultTimeout)) + context, err := interactive.SpawnShell(interactive.CreateGoExpectSpawner(), DefaultTimeout, interactive.Verbose(LogLevelTraceEnabled), interactive.SendTimeout(DefaultTimeout)) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(context).ToNot(gomega.BeNil()) gomega.Expect(context.GetExpecter()).ToNot(gomega.BeNil()) @@ -76,16 +79,23 @@ func logLevel() string { log.Info("LOG_LEVEL environment is not set, defaulting to DEBUG") logLevel = "debug" //nolint:goconst } + return logLevel } // SetLogLevel sets the log level for logrus based on the "LOG_LEVEL" environment variable func SetLogLevel() { var aLogLevel, err = log.ParseLevel(logLevel()) + if err != nil { log.Error("LOG_LEVEL environment set with an invalid value, defaulting to DEBUG \n Valid values are: trace, debug, info, warn, error, fatal, panic") aLogLevel = log.DebugLevel } + + if aLogLevel == log.TraceLevel { + LogLevelTraceEnabled = true + } + log.Info("Log level set to:", aLogLevel) log.SetLevel(aLogLevel) } diff --git a/test-network-function/suite_test.go b/test-network-function/suite_test.go index 81b467fb7..93a36d658 100644 --- a/test-network-function/suite_test.go +++ b/test-network-function/suite_test.go @@ -129,6 +129,9 @@ func TestTest(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) common.SetLogFormat() common.SetLogLevel() + if common.LogLevelTraceEnabled { + config.EnableExpectersVerboseMode() + } // Display GinkGo Version log.Info("Ginkgo Version: ", ginkgo.GINKGO_VERSION) // Display the latest previously released build in case this build is not released From 848d7e16ba5b4ba092340a3cced9768cad16285c Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Tue, 2 Nov 2021 11:15:01 -0500 Subject: [PATCH 123/344] Adding source for overview.svg picture (#430) --- docs/images/overview.drawio | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/images/overview.drawio diff --git a/docs/images/overview.drawio b/docs/images/overview.drawio new file mode 100644 index 000000000..0cdf73d0c --- /dev/null +++ b/docs/images/overview.drawio @@ -0,0 +1 @@ +7Vtbc6M2FP41fkxGIMDwmDiXPnS7mcl0dvPUUUDYajFihZzY/fWVuJiLZOILBidTMpOggxDwnU/nJmUCZ8v1I0PJ4hsNcDQxQbCewLuJaXquIX5LwSYX2LaXC+aMBLnIqATP5F9cCEEhXZEAp42OnNKIk6Qp9GkcY583ZIgx+t7sFtKo+dQEzbEiePZRpEp/kIAvcqlrg0r+GybzRflkAxRXlqjsXAjSBQroe00E7ydwxijl+dlyPcORxK7EJb/vYcfV7YsxHPN9bniMFn/NvqEfN97985/ceDR+3V1dmU4+zBuKVsUXF2/LNyUEjK7iAMtRwATevi8Ix88J8uXVd6FzIVvwZSRahjgNSRTNaERZdi8MEHZDX8hTzug/uHbF8V38GoorxQtgxvF656cZW8AE0TBdYs42okt5w7TAuCDZFvP3SmV2qbJFTV1meSMqaDLfjl0hKU4KMA8A9sy4hrb80eKaHfIOGvOaPD96wttt4Q1VvA1Phzc4F96mgndCAyEw+oU9DE1fS+fAeXVspx94zTa8jgrv1FbRtc4F7lQBd/bHgxBIIJm0yDjlCs7i+3kTzCZoMY1xC+FChCIyj0XTF3iJ8eGtRJMIm3xTXFiSIJCP0Wqvqd+h2A60bD+TPtyPjQuOgxvp/ipUA5QuMliMplak/AlxgXScSUwAt7oqXZ5A51agxTY/JazXdtl8KcbIGnfrAvO8tWloAAeKn23hL16frpiPO77bKgIAxOaYf2QMVH3WvYFGXaWM4Qhx8tZ8XZ0Oiyc8USI+ZEsXCzTpYrVpkH9mcVfdYbcHspsDwbazynFQBsootf3s41lmqLZTavG5aFLGF3ROYxTdV9LW7Kv6/E5pUpDlb8z5poj00IrTJh3xmvCfJZHE+UvFONGqOCYbmzrhfta5+NLk6Q5qtqw7ADbAoc66A2Dc3c76JbN3Gpn3ZulJlsZTLM0TYjzOrH7mYD+dc7VM77Kcq+GMOs22U+ulPrO6p9mBHmDUaVYmkh/OM2/MeWZ445LArLPAOAsLxtCpdaJOs1tFHIU2tQ6JdLhpRwDQ8ttlQLCvn7ec7v7QcLv6i5P8jXsNBmB3MFDEmL0x0pjs7/01ZsSeAq3hcREYi4rmuPYFKI7cl0lVKNIqTmicV9A4Iplrf8Br7Pfr210f6337q2vLCdCTbzeaEbPGt+vKEs7ZfDtUYFenUZWp+RFKU+L/HxAfY9THye7svrI7a+jszlKomZB4rtDzy5RxoGM1ANcG/kNWcQz7M2mAUZ57CngHgdeXSuymvTbtkVWiVjpJTDhBHE+2a0zkjfBNUfVML1RbfThT22spR1P21C6qnK3GDxS0h0iSqnzHPCjhqbKr6f7ZVY/+dP/QVM+DEz2l066DGkd6SmhOr+3uoXb4yoMzN7f1nPaS6kf93e7+pud19T9P5lZqrmbT2Cou63cPtbAfvKJUzJQvb9lgm5maSGDH8uW5LJvqdyamE0m45TJuQw/OrxUtL1ylmdW6ER0MN1lXF8XZXP79nuA4XZCQl8OJ18tHzK9/XSXbXlPJ20CirmQ4qJLVYnpNPWCJUp7NQjkjGY0i2egzBR91y0BbHVCnDlOjjrPV3Uvd11e1KcPfn4XsSXi+kLKlmCZoKWGNX9NEN12+jkIsXXg3rEI0tZEzlBh3lwurSO/EqklRZjymmDlG/AeBniknBoB2y81C98gAsB1JQm+/8K+32rdaxUu2BuKrB0u2tQP8mp2whnSjUN1ah+JNGc+K1xJvJRpZJeUD630ZSmpszOlDY7AV+egsuzGoyg7bsNRdBr8+YqnywDJ4j0a4ZOuxy88jVbXb+zSPrmrbA5vq7nX0vpcpD9k9oYk75NEv3cx96ZZ3HGs50rrc1eRauQ84jXLfNbDMDwxH1nrCjAiYpFsYTvHWvsHeqcW+0/Su5lwMJ5FwnCmWVuTE3drqxrGp96rbCRCG2MmWo3twt4Y7bQZIYORFDM3CXhYZrU8D9yK2wutW9AfdrWepuUCDwfFXYPDYK6OWSuGuePGTbHAfK7gzS/Ta6j00uBPPvnac5ljliu1A8Z3VvVn2MgKHw5KNYcIDuGd40MuOSZU5zrSbNntT0LOubVAdLWYbgqBTrzoGJuf0Ysk5IMnOT55j64hKMDFwcmq5F8uP8YxX/3SxW3QxjrU1bbr05utEs/q/77x79c/z8P4/ \ No newline at end of file From 4377b44df2534477717b0727c4a11c582772f52c Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 2 Nov 2021 12:56:48 -0400 Subject: [PATCH 124/344] Removed the unnecessary loop in the session monitoring routine (#431) --- .gitignore | 5 ++++- pkg/config/config.go | 13 +++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index b22df2f98..921d29596 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,7 @@ pkg/tnf/reel/mocks/mock_reel.go # NOTE: mock_expect.go is tracked, as it is generated from source externally. pkg/tnf/interactive/mocks/mock_spawner.go temp/ -/tnf \ No newline at end of file +/tnf +/all-releases.txt +/latest-release-tag.txt +/release-tag.txt diff --git a/pkg/config/config.go b/pkg/config/config.go index 50180cce1..7207427f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -111,14 +111,11 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio gomega.Expect(err).To(gomega.BeNil()) // Set up a go routine which reads from the error channel go func() { - for { - select { - case err := <-outCh: - log.Fatalf("OC session to container %s/%s is broken due to: %v, aborting the test run", oc.GetPodName(), oc.GetPodContainerName(), err) - case <-oc.GetDoneChannel(): - log.Infof("stop watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) - return - } + select { + case err := <-outCh: + log.Fatalf("OC session to container %s/%s is broken due to: %v, aborting the test run", oc.GetPodName(), oc.GetPodContainerName(), err) + case <-oc.GetDoneChannel(): + log.Infof("stop watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) } }() ocChan <- oc From 151a239a50c89d8fdc17b1ac1c39fd353dcf415e Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Tue, 2 Nov 2021 20:10:08 +0200 Subject: [PATCH 125/344] Consolidate all cmds into one tnf cmd (#428) * Consolidate all cmds into one tnf cmd * Consolidate all cmds into one tnf cmd * delete from make file * delete from make file * delete from make file * update new failes * update make * update make * update new comments changes * change package name * update the docker file * update the developing md Co-authored-by: Salaheddine Hamadi Co-authored-by: Jun Chen --- CATALOG.md | 38 +---- DEVELOPING.md | 12 +- Dockerfile | 1 + Makefile | 21 +-- cmd/catalog/cmd/data/TEST_CASE_CATALOG.md | 3 - cmd/catalog/cmd/doc.go | 18 --- cmd/catalog/doc.go | 18 --- cmd/catalog/main.go | 30 ---- cmd/claim/doc.go | 18 --- cmd/claim/main.go | 139 ------------------ cmd/generic/doc.go | 18 --- cmd/generic/main.go | 30 ---- cmd/gradetool/doc.go | 18 --- cmd/gradetool/main.go | 56 ------- cmd/tnf/addclaim/addclaim.go | 113 ++++++++++++++ .../data => tnf/generate/catalog}/INTRO.md | 2 +- .../TEST_CASE_BUILDING_BLOCKS_CATALOG.md | 2 +- cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md | 3 + .../generate/catalog/catalog.go} | 18 +-- cmd/tnf/generate/handler/handler.go | 137 +++++++++++++++++ cmd/tnf/grade/grade.go | 63 ++++++++ cmd/{generic/cmd => tnf/jsontest}/jsontest.go | 18 +-- cmd/tnf/main.go | 138 ++--------------- 23 files changed, 351 insertions(+), 563 deletions(-) delete mode 100644 cmd/catalog/cmd/data/TEST_CASE_CATALOG.md delete mode 100644 cmd/catalog/cmd/doc.go delete mode 100644 cmd/catalog/doc.go delete mode 100644 cmd/catalog/main.go delete mode 100644 cmd/claim/doc.go delete mode 100644 cmd/claim/main.go delete mode 100644 cmd/generic/doc.go delete mode 100644 cmd/generic/main.go delete mode 100644 cmd/gradetool/doc.go delete mode 100644 cmd/gradetool/main.go create mode 100644 cmd/tnf/addclaim/addclaim.go rename cmd/{catalog/cmd/data => tnf/generate/catalog}/INTRO.md (94%) rename cmd/{catalog/cmd/data => tnf/generate/catalog}/TEST_CASE_BUILDING_BLOCKS_CATALOG.md (94%) create mode 100644 cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md rename cmd/{catalog/cmd/generate.go => tnf/generate/catalog/catalog.go} (94%) create mode 100644 cmd/tnf/generate/handler/handler.go create mode 100644 cmd/tnf/grade/grade.go rename cmd/{generic/cmd => tnf/jsontest}/jsontest.go (97%) diff --git a/CATALOG.md b/CATALOG.md index 7d9053935..f7272c57b 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -2,11 +2,9 @@ test-network-function contains a variety of `Test Cases`, as well as `Test Case Building Blocks`. * Test Cases: Traditional JUnit testcases, which are specified internally using `Ginkgo.It`. Test cases often utilize several Test Case Building Blocks. -* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. -## Test Case Catalog +* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`.## Test Case Building Blocks Catalog -Test Cases are the specifications used to perform a meaningful test. Test cases may run once, or several times against several targets. CNF Certification includes a number of normative and informative tests to ensure CNFs follow best practices. Here is the list of available Test Cases: -### http://test-network-function.com/testcases/access-control/cluster-role-bindings +A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations:### http://test-network-function.com/testcases/access-control/cluster-role-bindings Property|Description ---|--- @@ -14,7 +12,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/cluster-role-bindings tests that a Pod does not specify ClusterRoleBindings. Result Type|normative Suggested Remediation|In most cases, Pod's should not have ClusterRoleBindings. The suggested remediation is to remove the need for ClusterRoleBindings, if possible. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.10 and 6.3.6 ### http://test-network-function.com/testcases/access-control/host-resource Property|Description @@ -23,7 +20,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/host-resource tests several aspects of CNF best practices, including: 1. The Pod does not have access to Host Node Networking. 2. The Pod does not have access to Host Node Ports. 3. The Pod cannot access Host Node IPC space. 4. The Pod cannot access Host Node PID space. 5. The Pod is not granted NET_ADMIN SCC. 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. 9. The Pod is not granted NET_RAW SCC. 10. The Pod is not granted IPC_LOCK SCC. Result Type|normative Suggested Remediation|Ensure that each Pod in the CNF abides by the suggested best practices listed in the test description. In some rare cases, not all best practices can be followed. For example, some CNFs may be required to run as root. Such exceptions should be handled on a case-by-case basis, and should provide a proper justification as to why the best practice(s) cannot be followed. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/namespace Property|Description @@ -32,7 +28,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/namespace tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace. Result Type|normative Suggested Remediation|Ensure that your CNF utilizes a CNF-specific namespace. Additionally, the CNF-specific namespace should not start with "openshift-", except in rare cases. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/pod-role-bindings Property|Description @@ -41,7 +36,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-role-bindings ensures that a CNF does not utilize RoleBinding(s) in a non-CNF Namespace. Result Type|normative Suggested Remediation|Ensure the CNF is not configured to use RoleBinding(s) in a non-CNF Namespace. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.5 ### http://test-network-function.com/testcases/access-control/pod-service-account Property|Description @@ -50,7 +44,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-service-account tests that each CNF Pod utilizes a valid Service Account. Result Type|normative Suggested Remediation|Ensure that the each CNF Pod is configured to use a valid Service Account -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.3 and 6.2.7 ### http://test-network-function.com/testcases/affiliated-certification/container-is-certified Property|Description @@ -59,7 +52,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/container-is-certified tests whether container images have passed the Red Hat Container Certification Program (CCP). Result Type|normative Suggested Remediation|Ensure that your container has passed the Red Hat Container Certification Program (CCP). -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.7 ### http://test-network-function.com/testcases/affiliated-certification/operator-is-certified Property|Description @@ -68,7 +60,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/operator-is-certified tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP). Result Type|normative Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP). -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/diagnostic/clusterversion Property|Description @@ -77,7 +68,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/crd-status Property|Description @@ -86,7 +76,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/crd-status checks that all CRDs have a status subresource specification. Result Type|informative Suggested Remediation|make sure that all the CRDs have a meaningful status specification. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description @@ -95,7 +84,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/extract-node-information extracts informational information about the cluster. Result Type|informative Suggested Remediation| -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/list-cni-plugins Property|Description @@ -104,7 +92,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/list-cni-plugins lists CNI plugins Result Type|normative Suggested Remediation| -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.4 and 6.3.7 ### http://test-network-function.com/testcases/diagnostic/nodes-hw-info Property|Description @@ -113,7 +100,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/nodes-hw-info list nodes HW info Result Type|normative Suggested Remediation| -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/container-shutdown Property|Description @@ -122,7 +108,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/container-shutdown Ensure that the containers lifecycle pre-stop management feature is configured. Result Type|normative Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connexion). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace perdiod. 3) K8s will clean the remaining processes using KILL signal. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-high-availability Property|Description @@ -131,7 +116,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-high-availability ensures that CNF Pods specify podAntiAffinity rules and replica value is set to more than 1. Result Type|informative Suggested Remediation|In high availability cases, Pod podAntiAffinity rule should be specified for pod scheduling and pod replica value is set to more than 1 . -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-owner-type Property|Description @@ -140,7 +124,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s). Result Type|normative Suggested Remediation|Deploy the CNF using ReplicaSet/StatefulSet. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.8 ### http://test-network-function.com/testcases/lifecycle/pod-recreation Property|Description @@ -149,7 +132,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that a CNF is configured to support High Availability. First, this test cordons and drains a Node that hosts the CNF Pod. Next, the test ensures that OpenShift can re-instantiate the Pod on another Node, and that the actual replica count matches the desired replica count. Result Type|normative Suggested Remediation|Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-scheduling Property|Description @@ -158,7 +140,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-scheduling ensures that CNF Pods do not specify nodeSelector or nodeAffinity. In most cases, Pods should allow for instantiation on any underlying Node. Result Type|informative Suggested Remediation|In most cases, Pod's should not specify their host Nodes through nodeSelector or nodeAffinity. However, there are cases in which CNFs require specialized hardware specific to a particular class of Node. As such, this test is purely informative, and will not prevent a CNF from being certified. However, one should have an appropriate justification as to why nodeSelector and/or nodeAffinity is utilized by a CNF. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period Property|Description @@ -167,7 +148,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period tests whether the terminationGracePeriod is CNF-specific, or if the default (30s) is utilized. This test is informative, and will not affect CNF Certification. In many cases, the default terminationGracePeriod is perfectly acceptable for a CNF. Result Type|informative Suggested Remediation|Choose a terminationGracePeriod that is appropriate for your given CNF. If the default (30s) is appropriate, then feel free to ignore this informative message. This test is meant to raise awareness around how Pods are terminated, and to suggest that a CNF is configured based on its requirements. In addition to a terminationGracePeriod, consider utilizing a termination hook in the case that your application requires special shutdown instructions. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/scaling Property|Description @@ -176,7 +156,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/scaling tests that CNF deployments support scale in/out operations. First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s. Result Type|normative Suggested Remediation|Make sure CNF deployments/replica sets can scale in/out successfully. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/icmpv4-connectivity Property|Description @@ -185,7 +164,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. Result Type|normative Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities). -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/service-type Property|Description @@ -194,7 +172,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative Suggested Remediation|Ensure Services are not configured to use NodePort(s). -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.1 ### http://test-network-function.com/testcases/operator/install-source Property|Description @@ -203,7 +180,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-source tests whether a CNF Operator is installed via OLM. Result Type|normative Suggested Remediation|Ensure that your Operator is installed via OLM. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/operator/install-status Property|Description @@ -212,7 +188,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. Result Type|normative Suggested Remediation|Ensure that your Operator abides by the Operator Best Practices mentioned in the description. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/platform-alteration/base-image Property|Description @@ -221,7 +196,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/base-image ensures that the Container Base Image is not altered post-startup. This test is a heuristic, and ensures that there are no changes to the following directories: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Result Type|normative Suggested Remediation|Ensure that Container applications do not modify the Container Base Image. In particular, ensure that the following directories are not modified: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Ensure that all required binaries are built directly into the container image, and are not installed post startup. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.2 ### http://test-network-function.com/testcases/platform-alteration/boot-params Property|Description @@ -230,7 +204,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/boot-params tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node. Result Type|normative Suggested Remediation|Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerformanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.13 and 6.2.14 ### http://test-network-function.com/testcases/platform-alteration/hugepages-config Property|Description @@ -239,7 +212,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/isredhat-release Property|Description @@ -248,7 +220,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release verifies if the container base image is redhat. Result Type|normative Suggested Remediation|build a new docker image that's based on UBI (redhat universal base image). -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/sysctl-config Property|Description @@ -257,7 +228,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that no one has changed the node's sysctl configs after the node was created, the tests works by checking if the sysctl configs are consistent with the MachineConfig CR which defines how the node should be configured Result Type|normative Suggested Remediation|You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel Property|Description @@ -266,13 +236,11 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel ensures that the Node(s) hosting CNFs do not utilize tainted kernels. This test case is especially important to support Highly Available CNFs, since when a CNF is re-instantiated on a backup Node, that Node's kernel may not have the same hacks.' Result Type|normative Suggested Remediation|Test failure indicates that the underlying Node's' kernel is tainted. Ensure that you have not altered underlying Node(s) kernels in order to run the CNF. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.14 ## Test Case Building Blocks Catalog -A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: -### http://test-network-function.com/tests/clusterVersion +A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations:### http://test-network-function.com/tests/clusterVersion Property|Description ---|--- Version|v1.0.0 diff --git a/DEVELOPING.md b/DEVELOPING.md index b9b5a1f4e..c6650454f 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -250,19 +250,13 @@ Now that you have a sample JSON test defined, you can go ahead and run your JSON In order to run the test, you must first make the jsontest CLI. Issue the following command: ```shell-script -make jsontest-cli -``` - -After that completes, issue the following command: - -```shell-script -./jsontest run shell examples/ping.json +./tnf jsontest shell examples/ping.json ``` You will get something similar to the following: ```shell-script -% ./jsontest-cli run shell examples/ping.json +% ./tnf jsontest shell examples/ping.json INFO[0000] Running examples/ping.json from a local shell context 2020/12/06 13:32:53 Sent: "ping -c 5 www.redhat.com\n" 2020/12/06 13:32:57 Match for RE: "(?m)(\\d+) packets transmitted, (\\d+)( packets){0,1} received, (?:\\+(\\d+) errors)?.*$" found: ["5 packets transmitted, 5 packets received, 0.0% packet loss" "5" "5" " packets" ""] Buffer: "PING e3396.dscx.akamaiedge.net (23.34.95.235): 56 data bytes\n64 bytes from 23.34.95.235: icmp_seq=0 ttl=59 time=17.661 ms\n64 bytes from 23.34.95.235: icmp_seq=1 ttl=59 time=25.993 ms\n64 bytes from 23.34.95.235: icmp_seq=2 ttl=59 time=26.353 ms\n64 bytes from 23.34.95.235: icmp_seq=3 ttl=59 time=25.725 ms\n64 bytes from 23.34.95.235: icmp_seq=4 ttl=59 time=22.403 ms\n\n--- e3396.dscx.akamaiedge.net ping statistics ---\n5 packets transmitted, 5 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 17.661/23.627/26.353/3.302 ms\n" @@ -325,7 +319,7 @@ Note that `testResult` is 1, indicating `tnf.SUCCESS`. If you wish to explore the `oc` and `ssh` variants of `jsontest-cli`, please consult the following: ```shell-script -./jsontest-cli run -h +./jsontest -h ``` ### Including a JSON-based test in a Ginkgo Test Suite diff --git a/Dockerfile b/Dockerfile index d6797658b..b89df6fdf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,7 @@ RUN make install-lint RUN make install-tools && \ make mocks && \ make update-deps && \ + make build-tnf-tool && \ make build-cnf-tests-debug # Extract what's needed to run at a seperate location diff --git a/Makefile b/Makefile index 40f9c3eeb..0aa6544df 100644 --- a/Makefile +++ b/Makefile @@ -21,10 +21,6 @@ GO_PACKAGES=$(shell go list ./... | grep -v vendor) clean \ lint \ test \ - build-jsontest-cli \ - build-gradetool \ - build-catalog-json \ - build-catalog-md \ build-cnf-tests \ run-cnf-tests \ run-generic-cnf-tests \ @@ -55,10 +51,9 @@ GOLANGCI_VERSION=v1.42.1 # Run the unit tests and build all binaries build: make test - make build-cnf-tests - make build-jsontest-cli - make build-gradetool make build-tnf-tool + make build-cnf-tests + build-tnf-tool: go build -o tnf -v cmd/tnf/main.go @@ -84,21 +79,13 @@ test: mocks go build ${COMMON_GO_ARGS} ./... go test -coverprofile=cover.out `go list ./... | grep -v "github.com/test-network-function/test-network-function/test-network-function" | grep -v mock` -# build the binary that can be used to run JSON-defined tests. -build-jsontest-cli: - go build -o jsontest-cli -v cmd/generic/main.go - -# build the binary that can be used to run gradetool. -build-gradetool: - go build -o gradetool -v cmd/gradetool/main.go - # generate the test catalog in JSON build-catalog-json: - go run cmd/catalog/main.go generate json > catalog.json + ./tnf generate catalog json > catalog.json # generate the test catalog in Markdown build-catalog-md: - go run cmd/catalog/main.go generate markdown > CATALOG.md + ./tnf generate catalog markdown > CATALOG.md # build the CNF test binary build-cnf-tests: diff --git a/cmd/catalog/cmd/data/TEST_CASE_CATALOG.md b/cmd/catalog/cmd/data/TEST_CASE_CATALOG.md deleted file mode 100644 index 9d87bd94a..000000000 --- a/cmd/catalog/cmd/data/TEST_CASE_CATALOG.md +++ /dev/null @@ -1,3 +0,0 @@ -## Test Case Catalog - -Test Cases are the specifications used to perform a meaningful test. Test cases may run once, or several times against several targets. CNF Certification includes a number of normative and informative tests to ensure CNFs follow best practices. Here is the list of available Test Cases: diff --git a/cmd/catalog/cmd/doc.go b/cmd/catalog/cmd/doc.go deleted file mode 100644 index e0dfed820..000000000 --- a/cmd/catalog/cmd/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -// Package cmd provides a CLI implementation for generating the test catalog in JSON or markdown. -package cmd diff --git a/cmd/catalog/doc.go b/cmd/catalog/doc.go deleted file mode 100644 index e43d69173..000000000 --- a/cmd/catalog/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -// Package main provides a CLI driven tool to generate the test catalog. -package main diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go deleted file mode 100644 index 1da5b9b28..000000000 --- a/cmd/catalog/main.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package main - -import ( - "log" - - "github.com/test-network-function/test-network-function/cmd/catalog/cmd" -) - -// main generates a JSON formatted version of the test catalog. -func main() { - if err := cmd.Execute(); err != nil { - log.Fatalf("Could not generate the test catalog: %s", err) - } -} diff --git a/cmd/claim/doc.go b/cmd/claim/doc.go deleted file mode 100644 index b050892a2..000000000 --- a/cmd/claim/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -// Package main provides a CLI driven tool to append test suite result to existing claim file. -package main diff --git a/cmd/claim/main.go b/cmd/claim/main.go deleted file mode 100644 index a31f536fd..000000000 --- a/cmd/claim/main.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package main - -import ( - "encoding/json" - "flag" - "fmt" - - "github.com/test-network-function/test-network-function-claim/pkg/claim" - "github.com/test-network-function/test-network-function/pkg/junit" - - "log" - "os" - "path/filepath" -) - -const ( - argsLen = 2 - claimAdd = "claim-add" - claimFilePermissions = 0644 -) - -var ( - // claim-add subcommand flag pointers - // Adding a new choice for --claimfile of 'substring' and a new --reportFiles flag - claimFileTextPtr *string - reportFilesTextPtr *string -) - -func main() { - // Subcommands - claimAddCommand := flag.NewFlagSet("claim-add", flag.ExitOnError) - - // claim-add subcommand flag pointers - // Adding a new choice for --claimfile of 'substring' and a new --reportFiles flag - claimFileTextPtr = claimAddCommand.String("claimfile", "", "existing claim file. (Required)") - reportFilesTextPtr = claimAddCommand.String("reportdir", "", "dir of JUnit XML reports. (Required)") - - // Verify that a subcommand has been provided - // os.Arg[0] is the main command - // os.Arg[1] will be the subcommand - if len(os.Args) < argsLen { - log.Fatalf("claim-add subcommand is required") - } - - // Switch on the subcommand - // Parse the flags for appropriate FlagSet - // FlagSet.Parse() requires a set of arguments to parse as input - // os.Args[2:] will be all arguments starting after the subcommand at os.Args[1] - switch os.Args[1] { - case claimAdd: - if err := claimAddCommand.Parse(os.Args[2:]); err != nil { - log.Fatalf("Error reading argument %v", err) - } - default: - flag.PrintDefaults() - os.Exit(1) - } - - if claimAddCommand.Parsed() { - // Required Flags - if *claimFileTextPtr == "" { - claimAddCommand.PrintDefaults() - os.Exit(1) - } - if *reportFilesTextPtr == "" { - claimAddCommand.PrintDefaults() - os.Exit(1) - } - claimUpdate() - } -} - -func claimUpdate() { - fileUpdated := false - dat, err := os.ReadFile(*claimFileTextPtr) - if err != nil { - log.Fatalf("Error reading claim file :%v", err) - } - - claimRoot := readClaim(&dat) - junitMap := claimRoot.Claim.RawResults - - items, _ := os.ReadDir(*reportFilesTextPtr) - - for _, item := range items { - fileName := item.Name() - extension := filepath.Ext(fileName) - reportKeyName := fileName[0 : len(fileName)-len(extension)] - - if _, ok := junitMap[reportKeyName]; ok { - log.Printf("Skipping: %s already exists in supplied `%s` claim file", reportKeyName, *claimFileTextPtr) - } else { - junitMap[reportKeyName], err = junit.ExportJUnitAsMap(fmt.Sprintf("%s/%s", *reportFilesTextPtr, item.Name())) - if err != nil { - log.Fatalf("Error reading JUnit XML file into JSON: %v", err) - } - fileUpdated = true - } - } - claimRoot.Claim.RawResults = junitMap - payload, err := json.MarshalIndent(claimRoot, "", " ") - if err != nil { - log.Fatalf("Failed to generate the claim: %v", err) - } - err = os.WriteFile(*claimFileTextPtr, payload, claimFilePermissions) - if err != nil { - log.Fatalf("Error writing claim data:\n%s", string(payload)) - } - if fileUpdated { - log.Printf("Claim file `%s` updated\n", *claimFileTextPtr) - } else { - log.Printf("No changes were applied to `%s`\n", *claimFileTextPtr) - } -} - -func readClaim(contents *[]byte) *claim.Root { - var claimRoot claim.Root - err := json.Unmarshal(*contents, &claimRoot) - if err != nil { - log.Fatalf("Error reading claim constents file into type: %v", err) - } - return &claimRoot -} diff --git a/cmd/generic/doc.go b/cmd/generic/doc.go deleted file mode 100644 index 8fe4ff063..000000000 --- a/cmd/generic/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -// Package main introduces an executable (jsontest) used to run JSON based tests. -package main diff --git a/cmd/generic/main.go b/cmd/generic/main.go deleted file mode 100644 index 91b7adc69..000000000 --- a/cmd/generic/main.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package main - -import ( - "log" - - "github.com/test-network-function/test-network-function/cmd/generic/cmd" -) - -func main() { - err := cmd.Execute() - if err != nil { - log.Fatalf("Fatal Error: %s", err) - } -} diff --git a/cmd/gradetool/doc.go b/cmd/gradetool/doc.go deleted file mode 100644 index e02028a86..000000000 --- a/cmd/gradetool/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -// Package main introduces an executable (gradetool) used proposing a grade for a CNF based on test results. -package main diff --git a/cmd/gradetool/main.go b/cmd/gradetool/main.go deleted file mode 100644 index 35dacff98..000000000 --- a/cmd/gradetool/main.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (C) 2020-2021 Red Hat, Inc. -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/test-network-function/test-network-function/pkg/gradetool" -) - -const ( - flagResultsPath = "results" - flagPolicyPath = "policy" - flagOutputPath = "o" -) - -func main() { - resultsPath := flag.String(flagResultsPath, "", "Path to the input test results file") - policyPath := flag.String(flagPolicyPath, "", "Path to the input policy file") - outputPath := flag.String(flagOutputPath, "", "Path to the output file") - flag.Parse() - if resultsPath == nil || *resultsPath == "" { - flag.Usage() - return - } - if policyPath == nil || *policyPath == "" { - flag.Usage() - return - } - if outputPath == nil || *outputPath == "" { - flag.Usage() - return - } - - err := gradetool.GenerateGrade(*resultsPath, *policyPath, *outputPath) - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/cmd/tnf/addclaim/addclaim.go b/cmd/tnf/addclaim/addclaim.go new file mode 100644 index 000000000..a7e03b66d --- /dev/null +++ b/cmd/tnf/addclaim/addclaim.go @@ -0,0 +1,113 @@ +package claim + +import ( + "fmt" + "os" + "path/filepath" + + "encoding/json" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/test-network-function/test-network-function-claim/pkg/claim" + "github.com/test-network-function/test-network-function/pkg/junit" +) + +var ( + Reportdir string + Claim string + + addcalim = &cobra.Command{ + Use: "claim", + Short: "The test suite generates a \"claim\" file", + RunE: claimUpdate, + } + claimAddFile = &cobra.Command{ + Use: "add", + Short: "The test suite generates a \"claim\" file", + RunE: claimUpdate, + } +) + +const ( + claimFilePermissions = 0644 +) + +func claimUpdate(cmd *cobra.Command, args []string) error { + claimFileTextPtr := &Claim + reportFilesTextPtr := &Reportdir + fileUpdated := false + dat, err := os.ReadFile(*claimFileTextPtr) + if err != nil { + log.Fatalf("Error reading claim file :%v", err) + } + + claimRoot := readClaim(&dat) + junitMap := claimRoot.Claim.RawResults + + items, _ := os.ReadDir(*reportFilesTextPtr) + + for _, item := range items { + fileName := item.Name() + extension := filepath.Ext(fileName) + reportKeyName := fileName[0 : len(fileName)-len(extension)] + + if _, ok := junitMap[reportKeyName]; ok { + log.Printf("Skipping: %s already exists in supplied `%s` claim file", reportKeyName, *claimFileTextPtr) + } else { + junitMap[reportKeyName], err = junit.ExportJUnitAsMap(fmt.Sprintf("%s/%s", *reportFilesTextPtr, item.Name())) + if err != nil { + log.Fatalf("Error reading JUnit XML file into JSON: %v", err) + } + fileUpdated = true + } + } + claimRoot.Claim.RawResults = junitMap + payload, err := json.MarshalIndent(claimRoot, "", " ") + if err != nil { + log.Fatalf("Failed to generate the claim: %v", err) + } + err = os.WriteFile(*claimFileTextPtr, payload, claimFilePermissions) + if err != nil { + log.Fatalf("Error writing claim data:\n%s", string(payload)) + } + if fileUpdated { + log.Printf("Claim file `%s` updated\n", *claimFileTextPtr) + } else { + log.Printf("No changes were applied to `%s`\n", *claimFileTextPtr) + } + + return nil +} + +func readClaim(contents *[]byte) *claim.Root { + var claimRoot claim.Root + err := json.Unmarshal(*contents, &claimRoot) + if err != nil { + log.Fatalf("Error reading claim constents file into type: %v", err) + } + return &claimRoot +} + +func NewCommand() *cobra.Command { + claimAddFile.Flags().StringVarP( + &Reportdir, "reportdir", "r", "", + "dir of JUnit XML reports. (Required)", + ) + + err := claimAddFile.MarkFlagRequired("reportdir") + if err != nil { + return nil + } + + claimAddFile.Flags().StringVarP( + &Claim, "claim", "c", "", + "existing claim file. (Required)", + ) + err = claimAddFile.MarkFlagRequired("claim") + if err != nil { + return nil + } + addcalim.AddCommand(claimAddFile) + return addcalim +} diff --git a/cmd/catalog/cmd/data/INTRO.md b/cmd/tnf/generate/catalog/INTRO.md similarity index 94% rename from cmd/catalog/cmd/data/INTRO.md rename to cmd/tnf/generate/catalog/INTRO.md index 7ca07b3a1..45dafde4f 100644 --- a/cmd/catalog/cmd/data/INTRO.md +++ b/cmd/tnf/generate/catalog/INTRO.md @@ -2,4 +2,4 @@ test-network-function contains a variety of `Test Cases`, as well as `Test Case Building Blocks`. * Test Cases: Traditional JUnit testcases, which are specified internally using `Ginkgo.It`. Test cases often utilize several Test Case Building Blocks. -* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. +* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. \ No newline at end of file diff --git a/cmd/catalog/cmd/data/TEST_CASE_BUILDING_BLOCKS_CATALOG.md b/cmd/tnf/generate/catalog/TEST_CASE_BUILDING_BLOCKS_CATALOG.md similarity index 94% rename from cmd/catalog/cmd/data/TEST_CASE_BUILDING_BLOCKS_CATALOG.md rename to cmd/tnf/generate/catalog/TEST_CASE_BUILDING_BLOCKS_CATALOG.md index 858823245..826f83544 100644 --- a/cmd/catalog/cmd/data/TEST_CASE_BUILDING_BLOCKS_CATALOG.md +++ b/cmd/tnf/generate/catalog/TEST_CASE_BUILDING_BLOCKS_CATALOG.md @@ -1,3 +1,3 @@ ## Test Case Building Blocks Catalog -A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: +A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: \ No newline at end of file diff --git a/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md b/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md new file mode 100644 index 000000000..826f83544 --- /dev/null +++ b/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md @@ -0,0 +1,3 @@ +## Test Case Building Blocks Catalog + +A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: \ No newline at end of file diff --git a/cmd/catalog/cmd/generate.go b/cmd/tnf/generate/catalog/catalog.go similarity index 94% rename from cmd/catalog/cmd/generate.go rename to cmd/tnf/generate/catalog/catalog.go index 348354032..602b21e03 100644 --- a/cmd/catalog/cmd/generate.go +++ b/cmd/tnf/generate/catalog/catalog.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package cmd +package catalog import ( "encoding/json" @@ -51,7 +51,7 @@ var ( introMDFile = path.Join(mdDirectory, introMDFilename) // mdDirectory is the path to the directory of files that contain static text for CATALOG.md. - mdDirectory = path.Join("cmd", "catalog", "cmd", "data") + mdDirectory = path.Join("cmd", "tnf", "generate", "catalog") // tccFile is the path to the file that contains the test case catalog section introductory text for CATALOG.md. tccFile = path.Join(mdDirectory, tccFilename) @@ -60,15 +60,9 @@ var ( // for CATALOG.md tccbbFile = path.Join(mdDirectory, tccbbFilename) - // rootCmd is the root of the "catalog" CLI program. - rootCmd = &cobra.Command{ - Use: "catalog", - Short: "A CLI for creating the test catalog.", - } - // generateCmd is the root of the "catalog generate" CLI program. generateCmd = &cobra.Command{ - Use: "generate", + Use: "catalog", Short: "Generates the test catalog", } @@ -145,7 +139,6 @@ func outputTestCases() { fmt.Fprintf(os.Stdout, "Description|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Description, "\n", " ")) fmt.Fprintf(os.Stdout, "Result Type|%s\n", identifiers.Catalog[k].Type) fmt.Fprintf(os.Stdout, "Suggested Remediation|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Remediation, "\n", " ")) - fmt.Fprintf(os.Stdout, "Best Practice Reference|%s\n", strings.ReplaceAll(identifiers.Catalog[k].BestPracticeReference, "\n", " ")) } fmt.Println() fmt.Println() @@ -213,8 +206,7 @@ func runGenerateJSONCmd(_ *cobra.Command, _ []string) error { } // Execute executes the "catalog" CLI. -func Execute() error { +func NewCommand() *cobra.Command { generateCmd.AddCommand(jsonGenerateCmd, markdownGenerateCmd) - rootCmd.AddCommand(generateCmd) - return rootCmd.Execute() + return generateCmd } diff --git a/cmd/tnf/generate/handler/handler.go b/cmd/tnf/generate/handler/handler.go new file mode 100644 index 000000000..e2a8c76e8 --- /dev/null +++ b/cmd/tnf/generate/handler/handler.go @@ -0,0 +1,137 @@ +package handler + +import ( + "bufio" + "os" + "path" + "strings" + "text/template" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type myHandler struct { + UpperHandlername string + LowerHandlername string +} + +const ( + envHandlersFolder = "TNF_HANDLERS_SRC" + docFileName = "doc.go" + handlerFolderPerms = 0755 +) + +var ( + handler = &cobra.Command{ + Use: "handler", + Short: "adding new handler.", + RunE: generateHandlerFiles, + } + defaultHandlersFolder = path.Join("pkg", "tnf", "handlers") +) + +func getHandlersDirectory() (string, error) { + handlersDirectory := os.Getenv(envHandlersFolder) + + if handlersDirectory == "" { + log.Warnf("Environment variable %s not set. Handlers base folder will be set to ./%s", + envHandlersFolder, defaultHandlersFolder) + + handlersDirectory = defaultHandlersFolder + } else { + log.Infof("Env var %s found. Handlers directory: %s", envHandlersFolder, handlersDirectory) + } + + // Convert to absolute path. + if !path.IsAbs(handlersDirectory) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + handlersDirectory = path.Join(cwd, handlersDirectory) + } + + return handlersDirectory, nil +} + +func generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory string, myhandler myHandler) error { + type fileToRender struct { + templatePath string + renderedFileName string + } + + filesToRender := []fileToRender{ + {templatePath: path.Join(handlerTemplatesDirectory, "doc.tmpl"), renderedFileName: docFileName}, + {templatePath: path.Join(handlerTemplatesDirectory, "handler_test.tmpl"), renderedFileName: myhandler.LowerHandlername + "_test.go"}, + {templatePath: path.Join(handlerTemplatesDirectory, "handler.tmpl"), renderedFileName: myhandler.LowerHandlername + ".go"}, + } + + for _, renderedFileName := range filesToRender { + if err := createfile(renderedFileName.templatePath, renderedFileName.renderedFileName, myhandler, newHandlerDirectory); err != nil { + log.Errorf("Unable to create rendered file %s on %s", renderedFileName, newHandlerDirectory) + return err + } + } + + return nil +} + +func generateHandlerFiles(cmd *cobra.Command, args []string) error { + handlername := args[0] + myhandler := myHandler{LowerHandlername: strings.ToLower(handlername), UpperHandlername: strings.Title(handlername)} + + handlersDirectory, err := getHandlersDirectory() + if err != nil { + log.Fatalf("Unable to get handlers path.") + return err + } + + handlerTemplatesDirectory := path.Join(handlersDirectory, "handler_template") + + log.Infof("Using absolute path for tnf handlers directory: %s", handlersDirectory) + newHandlerDirectory := path.Join(handlersDirectory, myhandler.LowerHandlername) + + err = os.Mkdir(newHandlerDirectory, handlerFolderPerms) + if err != nil { + log.Fatal("Unable to create handler directory " + newHandlerDirectory) + os.Exit(1) + } + + err = generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory, myhandler) + if err != nil { + return err + } + + log.Infof("Handler files for %s successfully created in %s\n", myhandler.UpperHandlername, path.Join(newHandlerDirectory)) + return nil +} + +func createfile(templateFilePath, outputFileName string, myhandler myHandler, newHandlerDirectory string) error { + ftpl, err := template.ParseFiles(templateFilePath) + if err != nil { + return err + } + + temp := path.Join(newHandlerDirectory, outputFileName) + f, err := os.Create(temp) + if err != nil { + return err + } + + defer f.Close() + w := bufio.NewWriter(f) + + err = ftpl.Execute(w, myhandler) + if err != nil { + return err + } + w.Flush() + + return nil +} + +func NewCommand() *cobra.Command { + return handler +} diff --git a/cmd/tnf/grade/grade.go b/cmd/tnf/grade/grade.go new file mode 100644 index 000000000..3255011d2 --- /dev/null +++ b/cmd/tnf/grade/grade.go @@ -0,0 +1,63 @@ +package grade + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/test-network-function/test-network-function/pkg/gradetool" +) + +var ( + results string + policy string + OutputPath string + + grade = &cobra.Command{ + Use: "gradetool", + Short: "gradetool", + RunE: runGradetool, + } +) + +func runGradetool(cmd *cobra.Command, args []string) error { + resultsPath := results + policyPath := policy + outputPath := OutputPath + + err := gradetool.GenerateGrade(resultsPath, policyPath, outputPath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + return nil +} + +func NewCommand() *cobra.Command { + grade.Flags().StringVarP( + &results, "results", "r", "", + "Path to the input test results file", + ) + + grade.Flags().StringVarP( + &policy, "policy", "p", "", + "Path to the input policy file", + ) + grade.Flags().StringVarP( + &OutputPath, "OutputPath", "o", "", + "Path to the output file", + ) + err := grade.MarkFlagRequired("results") + if err != nil { + return nil + } + err = grade.MarkFlagRequired("policy") + if err != nil { + return nil + } + err = grade.MarkFlagRequired("OutputPath") + if err != nil { + return nil + } + return grade +} diff --git a/cmd/generic/cmd/jsontest.go b/cmd/tnf/jsontest/jsontest.go similarity index 97% rename from cmd/generic/cmd/jsontest.go rename to cmd/tnf/jsontest/jsontest.go index b8f335f53..4092c1d13 100644 --- a/cmd/generic/cmd/jsontest.go +++ b/cmd/tnf/jsontest/jsontest.go @@ -14,7 +14,7 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -package cmd +package jsontest import ( "encoding/json" @@ -88,18 +88,13 @@ const ( var ( // rootCmd is the jsontest executable root. Currently, the jsontest entrypoint has only one sub-command called // "run", which runs a generic JSON test. - rootCmd = &cobra.Command{ - Use: "jsontest-cli", + runCmd = &cobra.Command{ + Use: "jsontest", Short: "A CLI for creating, validating, and running JSON test-network-function tests.", - Long: `jsontest is a CLI library included in test-network-function used to prototype JSON test cases.`, + Long: `jsontest is a CLI library included in test-network-function used to prototype JSON test cases. The JSON test case can be run using oc, ssh, or local shell.`, } // runCmd is the json test executable option to run a JSON test. - runCmd = &cobra.Command{ - Use: "run", - Short: "run a JSON test case", - Long: `run is a CLI library included in test-network-function used to run a JSON test case. The JSON test case can be run using oc, ssh, or local shell.`, - } // shellCmd is the entrypoint for running a test case on the local shell. shellCmd = &cobra.Command{ @@ -353,8 +348,7 @@ func runPTYTemplateCmd(_ *cobra.Command, args []string) { } // Execute executes the jsontest program, returning any applicable errors. -func Execute() error { +func NewCommand() *cobra.Command { runCmd.AddCommand(ocCmd, sshCmd, shellCmd, ptyCmd, ptyTemplateCmd) - rootCmd.AddCommand(runCmd) - return rootCmd.Execute() + return runCmd } diff --git a/cmd/tnf/main.go b/cmd/tnf/main.go index a516119ee..5664b12ff 100644 --- a/cmd/tnf/main.go +++ b/cmd/tnf/main.go @@ -1,151 +1,35 @@ package main import ( - "bufio" - "os" - "path" - "strings" - "text/template" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" -) - -type myHandler struct { - UpperHandlername string - LowerHandlername string -} -const ( - envHandlersFolder = "TNF_HANDLERS_SRC" - docFileName = "doc.go" - handlerFolderPerms = 0755 + claim "github.com/test-network-function/test-network-function/cmd/tnf/addclaim" + "github.com/test-network-function/test-network-function/cmd/tnf/generate/catalog" + "github.com/test-network-function/test-network-function/cmd/tnf/generate/handler" + "github.com/test-network-function/test-network-function/cmd/tnf/grade" + "github.com/test-network-function/test-network-function/cmd/tnf/jsontest" ) var ( rootCmd = &cobra.Command{ Use: "tnf", - Short: "A CLI for creating, validating, and test-network-function tests.", + Short: "A CLI for creating, validating , and test-network-function tests.", } generate = &cobra.Command{ Use: "generate", Short: "generator tool for various tnf artifacts.", } - - handler = &cobra.Command{ - Use: "handler", - Short: "adding new handler.", - RunE: generateHandlerFiles, - } - - defaultHandlersFolder = path.Join("pkg", "tnf", "handlers") ) -func getHandlersDirectory() (string, error) { - handlersDirectory := os.Getenv(envHandlersFolder) - - if handlersDirectory == "" { - log.Warnf("Environment variable %s not set. Handlers base folder will be set to ./%s", - envHandlersFolder, defaultHandlersFolder) - - handlersDirectory = defaultHandlersFolder - } else { - log.Infof("Env var %s found. Handlers directory: %s", envHandlersFolder, handlersDirectory) - } - - // Convert to absolute path. - if !path.IsAbs(handlersDirectory) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - - handlersDirectory = path.Join(cwd, handlersDirectory) - } - - return handlersDirectory, nil -} - -func generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory string, myhandler myHandler) error { - type fileToRender struct { - templatePath string - renderedFileName string - } - - filesToRender := []fileToRender{ - {templatePath: path.Join(handlerTemplatesDirectory, "doc.tmpl"), renderedFileName: docFileName}, - {templatePath: path.Join(handlerTemplatesDirectory, "handler_test.tmpl"), renderedFileName: myhandler.LowerHandlername + "_test.go"}, - {templatePath: path.Join(handlerTemplatesDirectory, "handler.tmpl"), renderedFileName: myhandler.LowerHandlername + ".go"}, - } - - for _, renderedFileName := range filesToRender { - if err := createfile(renderedFileName.templatePath, renderedFileName.renderedFileName, myhandler, newHandlerDirectory); err != nil { - log.Errorf("Unable to create rendered file %s on %s", renderedFileName, newHandlerDirectory) - return err - } - } - - return nil -} - -func generateHandlerFiles(cmd *cobra.Command, args []string) error { - handlername := args[0] - myhandler := myHandler{LowerHandlername: strings.ToLower(handlername), UpperHandlername: strings.Title(handlername)} - - handlersDirectory, err := getHandlersDirectory() - if err != nil { - log.Fatalf("Unable to get handlers path.") - return err - } - - handlerTemplatesDirectory := path.Join(handlersDirectory, "handler_template") - - log.Infof("Using absolute path for tnf handlers directory: %s", handlersDirectory) - newHandlerDirectory := path.Join(handlersDirectory, myhandler.LowerHandlername) - - err = os.Mkdir(newHandlerDirectory, handlerFolderPerms) - if err != nil { - log.Fatal("Unable to create handler directory " + newHandlerDirectory) - os.Exit(1) - } - - err = generateHandlerFilesFromTemplates(handlerTemplatesDirectory, newHandlerDirectory, myhandler) - if err != nil { - return err - } - - log.Infof("Handler files for %s successfully created in %s\n", myhandler.UpperHandlername, path.Join(newHandlerDirectory)) - return nil -} - -func createfile(templateFilePath, outputFileName string, myhandler myHandler, newHandlerDirectory string) error { - ftpl, err := template.ParseFiles(templateFilePath) - if err != nil { - return err - } - - temp := path.Join(newHandlerDirectory, outputFileName) - f, err := os.Create(temp) - if err != nil { - return err - } - - defer f.Close() - w := bufio.NewWriter(f) - - err = ftpl.Execute(w, myhandler) - if err != nil { - return err - } - w.Flush() - - return nil -} - func main() { + rootCmd.AddCommand(claim.NewCommand()) rootCmd.AddCommand(generate) - generate.AddCommand(handler) + generate.AddCommand(catalog.NewCommand()) + generate.AddCommand(handler.NewCommand()) + rootCmd.AddCommand(jsontest.NewCommand()) + rootCmd.AddCommand(grade.NewCommand()) if err := rootCmd.Execute(); err != nil { log.Fatal(err) } From c9b90f3b5f417ebeedaaa8c81b57d6d93278585c Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 2 Nov 2021 15:56:21 -0400 Subject: [PATCH 126/344] Skip connectivity test when no suitable container found (#434) --- test-network-function/networking/suite.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 15a379390..eeb4c664a 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -79,10 +79,12 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { if env.TestOrchestrator == nil { ginkgo.Skip("Orchestrator is not deployed, skip this test") } + found := false for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue } + found = true context := cut.Oc testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), @@ -94,6 +96,9 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { testOrchestrator.DefaultNetworkIPAddress)) testPing(context, testOrchestrator.DefaultNetworkIPAddress, count) } + if !found { + ginkgo.Skip("No container found suitable for connectivity test") + } }) }) } From b755f7f17de85b560ac11a9f31ba934bfd59ed88 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 2 Nov 2021 16:59:01 -0400 Subject: [PATCH 127/344] Fix makefile target dependency (#435) * Fix makefile target dependency * Remove outdated targets --- Dockerfile | 1 - Makefile | 22 +++------------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index b89df6fdf..d6797658b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,6 @@ RUN make install-lint RUN make install-tools && \ make mocks && \ make update-deps && \ - make build-tnf-tool && \ make build-cnf-tests-debug # Extract what's needed to run at a seperate location diff --git a/Makefile b/Makefile index 0aa6544df..08cc7dc14 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,6 @@ GOLANGCI_VERSION=v1.42.1 # Run the unit tests and build all binaries build: make test - make build-tnf-tool make build-cnf-tests @@ -69,6 +68,7 @@ clean: make clean-mocks rm -f ./test-network-function/test-network-function.test rm -f ./test-network-function/cnf-certification-tests_junit.xml + rm -f ./tnf # Run configured linters lint: @@ -80,11 +80,11 @@ test: mocks go test -coverprofile=cover.out `go list ./... | grep -v "github.com/test-network-function/test-network-function/test-network-function" | grep -v mock` # generate the test catalog in JSON -build-catalog-json: +build-catalog-json: build-tnf-tool ./tnf generate catalog json > catalog.json # generate the test catalog in Markdown -build-catalog-md: +build-catalog-md: build-tnf-tool ./tnf generate catalog markdown > CATALOG.md # build the CNF test binary @@ -96,22 +96,6 @@ build-cnf-tests-debug: PATH=${PATH}:${GOBIN} ginkgo build -gcflags "all=-N -l" -ldflags "-X github.com/test-network-function/test-network-function/test-network-function.GitCommit=${GIT_COMMIT} -X github.com/test-network-function/test-network-function/test-network-function.GitRelease=${GIT_RELEASE} -X github.com/test-network-function/test-network-function/test-network-function.GitPreviousRelease=${GIT_PREVIOUS_RELEASE} -extldflags '-z relro -z now'" ./test-network-function make build-catalog-md -# run all CNF tests -run-cnf-tests: build-cnf-tests - ./run-cnf-suites.sh diagnostic generic multus operator container - -# run only the generic CNF tests -run-generic-cnf-tests: build-cnf-tests - ./run-cnf-suites.sh diagnostic generic - -# Run operator CNF tests -run-operator-tests: build-cnf-tests - ./run-cnf-suites.sh diagnostic operator - -# Run container CNF tests -run-container-tests: build-cnf-tests - ./run-cnf-suites.sh diagnostic container - # Each mock depends on one source file pkg/tnf/interactive/mocks/mock_spawner.go: pkg/tnf/interactive/spawner.go mockgen -source=pkg/tnf/interactive/spawner.go -destination=pkg/tnf/interactive/mocks/mock_spawner.go From 7094a234d907972490e445254878a64f32f2a39a Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 3 Nov 2021 08:22:45 -0500 Subject: [PATCH 128/344] Fix typos (#432) Co-authored-by: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> --- CATALOG.md | 2 +- test-network-function/identifiers/identifiers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index f7272c57b..1fe7e83cb 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -107,7 +107,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/container-shutdown Ensure that the containers lifecycle pre-stop management feature is configured. Result Type|normative -Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connexion). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace perdiod. 3) K8s will clean the remaining processes using KILL signal. +Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connection). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace period. 3) K8s will clean the remaining processes using KILL signal. ### http://test-network-function.com/testcases/lifecycle/pod-high-availability Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 69f02a2e5..22453e303 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -520,7 +520,7 @@ the changes for you.`, `Ensure that the containers lifecycle pre-stop management feature is configured.`), Remediation: ` It's considered best-practices to define prestop for proper management of container lifecycle. - The prestop can be used to gracefully stop the container and clean resources (e.g., DB connexion). + The prestop can be used to gracefully stop the container and clean resources (e.g., DB connection). The prestop can be configured using : 1) Exec : executes the supplied command inside the container @@ -528,7 +528,7 @@ the changes for you.`, When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. - 2) K8s will wait for a grace perdiod. + 2) K8s will wait for a grace period. 3) K8s will clean the remaining processes using KILL signal. `, BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", From 742a2d4bdeca1cff65ef203b197e4ef9e86a06a7 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 3 Nov 2021 15:21:18 +0100 Subject: [PATCH 129/344] Created generic command launcher on utils package. (#433) Co-authored-by: Jun Chen --- pkg/config/autodiscover/autodiscover.go | 62 ++----------------- pkg/config/autodiscover/autodiscover_debug.go | 13 ++-- .../autodiscover/autodiscover_targets.go | 14 ++--- pkg/config/autodiscover/csv_info.go | 7 +-- pkg/config/autodiscover/deployment_info.go | 19 +++--- .../autodiscover/deployment_info_test.go | 9 ++- pkg/config/autodiscover/pod_info.go | 8 +-- pkg/utils/utils.go | 54 +++++++++++++++- test-network-function/platform/suite.go | 39 +----------- 9 files changed, 87 insertions(+), 138 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index eb6ba7127..481b0ccdf 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -19,16 +19,13 @@ package autodiscover import ( "fmt" "os" - "path" "strconv" "time" - "github.com/onsi/gomega" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" - "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" + "github.com/test-network-function/test-network-function/pkg/utils" ) const ( @@ -42,19 +39,6 @@ const ( ) var ( - // PathRelativeToRoot is used to calculate relative filepaths for the `test-network-function` executable entrypoint. - pathRelativeToRoot = path.Join("..") - // TestFile is the file location of the command.json test case relative to the project root. - TestFile = path.Join("pkg", "tnf", "handlers", "command", "command.json") - // RelativeSchemaPath is the relative path to the generic-test.schema.json JSON schema. - relativeSchemaPath = path.Join(pathRelativeToRoot, schemaPath) - // pathToTestFile is the relative path to the command.json test case. - pathToTestFile = path.Join(pathRelativeToRoot, TestFile) - // schemaPath is the path to the generic-test.schema.json JSON schema relative to the project root. - schemaPath = path.Join("schemas", "generic-test.schema.json") - // commandDriver stores the csi driver JSON output. - commandDriver = make(map[string]interface{}) - expectersVerboseModeEnabled = false ) @@ -83,46 +67,12 @@ func buildLabelQuery(label configsections.Label) string { return fullLabelName } -func executeOcGetCommand(resourceType, labelQuery, namespace string) (string, error) { +func executeOcGetCommand(resourceType, labelQuery, namespace string) string { ocCommandToExecute := fmt.Sprintf(ocCommand, resourceType, namespace, labelQuery) - match, err := executeOcCommand(ocCommandToExecute) - if err != nil { - log.Error("can't run command, ", ocCommandToExecute, "Error=", err) - return "", err - } - err = jsonUnmarshal([]byte(match), &commandDriver) - gomega.Expect(err).To(gomega.BeNil()) - return match, err -} - -func executeOcCommand(command string) (string, error) { - values := make(map[string]interface{}) - values["COMMAND"] = command - values["TIMEOUT"] = ocCommandTimeOut.Nanoseconds() - context := interactive.GetContext(expectersVerboseModeEnabled) - tester, handler, result, err := generic.NewGenericFromMap(pathToTestFile, relativeSchemaPath, values) - - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handler).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - - test, err := tnf.NewTest(context.GetExpecter(), *tester, handler, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - if err != nil { - return "", err - } - test.RunAndValidate() - - genericTest := (*tester).(*generic.Generic) - gomega.Expect(genericTest).ToNot(gomega.BeNil()) - - matches := genericTest.Matches - gomega.Expect(len(matches)).To(gomega.Equal(1)) - match := genericTest.GetMatches()[0] - return match.Match, nil + match := utils.ExecuteCommand(ocCommandToExecute, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { + log.Error("can't run command: ", ocCommandToExecute) + }) + return match } // getContainersByLabel builds `config.Container`s from containers in pods matching a label. diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go index 82a999003..7c0cdb65b 100644 --- a/pkg/config/autodiscover/autodiscover_debug.go +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -27,6 +27,7 @@ import ( ds "github.com/test-network-function/test-network-function/pkg/tnf/handlers/daemonset" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/test-network-function/test-network-function/pkg/utils" ) const ( @@ -60,22 +61,18 @@ func FindDebugPods(tp *configsections.TestPartner) { func AddDebugLabel(nodeName string) { log.Info("add label", nodeLabelName, "=", nodeLabelValue, " to node ", nodeName) ocCommand := fmt.Sprintf(addlabelCommand, nodeName, nodeLabelName, nodeLabelValue) - _, err := executeOcCommand(ocCommand) - if err != nil { + _ = utils.ExecuteCommand(ocCommand, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { log.Error("error in adding label to node ", nodeName) - return - } + }) } // AddDebugLabel remove debug label from node func DeleteDebugLabel(nodeName string) { log.Info("delete label", nodeLabelName, "=", nodeLabelValue, "to node ", nodeName) ocCommand := fmt.Sprintf(deletelabelCommand, nodeName, nodeLabelName) - _, err := executeOcCommand(ocCommand) - if err != nil { + _ = utils.ExecuteCommand(ocCommand, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { log.Error("error in removing label from node ", nodeName) - return - } + }) } // CheckDebugDaemonset checks if the debug pods are deployed properly diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 891fb4d13..7046fc9d5 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -18,7 +18,6 @@ package autodiscover import ( "encoding/json" - "os/exec" "strings" "time" @@ -29,6 +28,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/interactive" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/pkg/utils" ) const ( @@ -205,16 +205,12 @@ func getConfiguredOperatorTests() (opTests []string) { // getClusterCrdNames returns a list of crd names found in the cluster. func getClusterCrdNames() ([]string, error) { - // ToDo: Use command handler. - cmd := exec.Command("bash", "-c", ocGetClusterCrdNamesCommand) - - out, err := cmd.Output() - if err != nil { - return nil, err - } + out := utils.ExecuteCommand(ocGetClusterCrdNamesCommand, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { + log.Error("can't run command: ", ocGetClusterCrdNamesCommand) + }) var crdNamesList []string - err = json.Unmarshal(out, &crdNamesList) + err := json.Unmarshal([]byte(out), &crdNamesList) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index 74fcdfa6d..78ae2324a 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -70,16 +70,13 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error // GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that // label will be returned, regardless of the labels value. func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { - out, err := executeOcGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) + out := executeOcGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) - if err != nil { - return nil, err - } log.Debug("JSON output for all pods labeled with: ", labelName) log.Debug("Command: ", out) var csvList CSVList - err = jsonUnmarshal([]byte(out), &csvList) + err := jsonUnmarshal([]byte(out), &csvList) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index 7728beed1..03795089c 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -19,9 +19,11 @@ package autodiscover import ( "encoding/json" "fmt" - "os/exec" + log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" + "github.com/test-network-function/test-network-function/pkg/utils" ) const ( @@ -30,8 +32,10 @@ const ( var ( jsonUnmarshal = json.Unmarshal - execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { - return cmd.Output() + execCommandOutput = func(command string) string { + return utils.ExecuteCommand(command, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { + log.Error("can't run command: ", command) + }) } ) @@ -80,15 +84,10 @@ func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsection jqArgs := fmt.Sprintf("'[.items[] | select(.spec.template.metadata.labels.%s)]'", labelQuery) ocCmd := fmt.Sprintf("oc get %s -n %s -o json | jq %s", resourceTypeDeployment, namespace, jqArgs) - cmd := exec.Command("bash", "-c", ocCmd) - - out, err := execCommandOutput(cmd) - if err != nil { - return nil, err - } + out := execCommandOutput(ocCmd) var deploymentList DeploymentList - err = jsonUnmarshal(out, &deploymentList.Items) + err := jsonUnmarshal([]byte(out), &deploymentList.Items) if err != nil { return nil, err } diff --git a/pkg/config/autodiscover/deployment_info_test.go b/pkg/config/autodiscover/deployment_info_test.go index c94f18d72..1c6b8fb3a 100644 --- a/pkg/config/autodiscover/deployment_info_test.go +++ b/pkg/config/autodiscover/deployment_info_test.go @@ -21,7 +21,6 @@ import ( "errors" "log" "os" - "os/exec" "path" "testing" @@ -94,14 +93,14 @@ func TestGetTargetDeploymentByNamespace(t *testing.T) { for _, tc := range testCases { // Setup the mock functions if tc.badExec { - execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { - return nil, tc.execErr + execCommandOutput = func(command string) string { + return "" } } else { - execCommandOutput = func(cmd *exec.Cmd) ([]byte, error) { + execCommandOutput = func(command string) string { contents, err := os.ReadFile(testJQFilePath) assert.Nil(t, err) - return contents, nil + return string(contents) } } if tc.badJSONUnmarshal { diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 43355ef3d..81ffd3e0b 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -152,17 +152,13 @@ func (pr *PodResource) annotationUnmarshalError(annotationKey string, err error) // GetPodsByLabel will return all pods with a given label value. If `labelValue` is an empty string, all pods with that // label will be returned, regardless of the labels value. func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, error) { - out, err := executeOcGetCommand(resourceTypePods, buildLabelQuery(label), namespace) - - if err != nil { - return nil, err - } + out := executeOcGetCommand(resourceTypePods, buildLabelQuery(label), namespace) log.Debug("JSON output for all pods labeled with: ", label) log.Debug("Command: ", out) var podList PodList - err = jsonUnmarshal([]byte(out), &podList) + err := jsonUnmarshal([]byte(out), &podList) if err != nil { return nil, err } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 9ce912021..a3c220125 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -3,10 +3,25 @@ package utils import ( "errors" "os" + "path" "path/filepath" "strings" + "time" + "github.com/onsi/gomega" log "github.com/sirupsen/logrus" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/tnf/interactive" +) + +var ( + // pathRelativeToRoot is used to calculate relative filepaths to the tnf folder. + pathRelativeToRoot = path.Join("..") + // commandHandlerFilePath is the file location of the command handler. + commandHandlerFilePath = path.Join(pathRelativeToRoot, "pkg", "tnf", "handlers", "command", "command.json") + // handlerJSONSchemaFilePath is the file location of the json handlers generic schema. + handlerJSONSchemaFilePath = path.Join(pathRelativeToRoot, "schemas", "generic-test.schema.json") ) // ArgListToMap takes a list of strings of the form "key=value" and translate it into a map @@ -35,9 +50,9 @@ func FilterArray(vs []string, f func(string) bool) []string { return vsf } -func CheckFileExists(path, name string) { - fullPath, _ := filepath.Abs(path) - if _, err := os.Stat(path); err == nil { +func CheckFileExists(filePath, name string) { + fullPath, _ := filepath.Abs(filePath) + if _, err := os.Stat(filePath); err == nil { log.Infof("Path to %s file found and valid: %s ", name, fullPath) } else if errors.Is(err, os.ErrNotExist) { log.Fatalf("Path to %s file not found: %s , Exiting", name, fullPath) @@ -45,3 +60,36 @@ func CheckFileExists(path, name string) { log.Infof("Path to %s file not valid: %s , err=%s, exiting", name, fullPath, err) } } + +// ExecuteCommand uses the generic command handler to execute an arbitrary interactive command, returning +// its output wihout any other check. +func ExecuteCommand(command string, timeout time.Duration, context *interactive.Context, failureCallbackFun func()) string { + log.Debugf("Executing command: %s", command) + + values := make(map[string]interface{}) + // Escapes the double quote char to make a valid json string. + values["COMMAND"] = strings.ReplaceAll(command, "\"", "\\\"") + values["TIMEOUT"] = timeout.Nanoseconds() + + tester, handler, result, err := generic.NewGenericFromMap(commandHandlerFilePath, handlerJSONSchemaFilePath, values) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handler).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + test, err := tnf.NewTest(context.GetExpecter(), *tester, handler, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + test.RunAndValidateWithFailureCallback(failureCallbackFun) + + genericTest := (*tester).(*generic.Generic) + gomega.Expect(genericTest).ToNot(gomega.BeNil()) + + matches := genericTest.Matches + gomega.Expect(len(matches)).To(gomega.Equal(1)) + match := genericTest.GetMatches()[0] + return match.Match +} diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index fd3cfc53a..b61b20a28 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -19,7 +19,6 @@ package platform import ( "encoding/json" "fmt" - "path" "regexp" "sort" "strconv" @@ -41,7 +40,6 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/cnffsdiff" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/containerid" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/currentkernelcmdlineargs" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/mckernelarguments" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodemcname" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/nodetainted" @@ -61,11 +59,7 @@ const ( HugepageszParam = "hugepagesz" DefaultHugepagesz = "default_hugepagesz" KernArgsKeyValueSplitLen = 2 -) - -var ( - commandHandlerFilePath = path.Join(common.PathRelativeToRoot, "pkg", "tnf", "handlers", "command", "command.json") - mcGetterCommandTimeout = time.Second * 30 + commandTimeout = 30 * time.Second ) type hugePagesConfig struct { @@ -420,33 +414,6 @@ func testTainted(env *config.TestEnvironment) { }) } -func runAndValidateCommand(command string, context *interactive.Context, failureCallbackFun func()) (match string) { - log.Debugf("Launching generic command handler for cmd: %s", command) - - values := make(map[string]interface{}) - values["COMMAND"] = command - values["TIMEOUT"] = mcGetterCommandTimeout.Nanoseconds() - - tester, handlers, result, err := generic.NewGenericFromMap(commandHandlerFilePath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - - test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) - gomega.Expect(test).ToNot(gomega.BeNil()) - gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidateWithFailureCallback(failureCallbackFun) - - genericTest := (*tester).(*generic.Generic) - gomega.Expect(genericTest).ToNot(gomega.BeNil()) - - matches := genericTest.Matches - gomega.Expect(len(matches)).To(gomega.Equal(1)) - return genericTest.GetMatches()[0].Match -} - func hugepageSizeToInt(s string) int { num, _ := strconv.Atoi(s[:len(s)-1]) unit := s[len(s)-1] @@ -523,7 +490,7 @@ func getNodeNumaHugePages(node *config.NodeConfig) (hugepages numaHugePagesPerSi // This command must run inside the node, so we'll need the node's context to run commands inside the debug daemonset pod. context := interactive.NewContext(node.Oc.GetExpecter(), node.Oc.GetErrorChannel()) var commandErr error - hugepagesCmdOut := runAndValidateCommand(cmd, context, func() { + hugepagesCmdOut := utils.ExecuteCommand(cmd, commandTimeout, context, func() { commandErr = fmt.Errorf("failed to get node %s hugepages per numa", node.Name) }) if commandErr != nil { @@ -565,7 +532,7 @@ func getMachineConfig(mcName string) (machineConfig, error) { // Local shell context is needed for the command handler. context := common.GetContext() - mcJSON := runAndValidateCommand(fmt.Sprintf("oc get mc %s -o json", mcName), context, func() { + mcJSON := utils.ExecuteCommand(fmt.Sprintf("oc get mc %s -o json", mcName), commandTimeout, context, func() { commandErr = fmt.Errorf("failed to get json machineconfig %s", mcName) }) if commandErr != nil { From 734a116097f807c67adbba8856f3dc74d819ef6e Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 3 Nov 2021 12:07:29 -0400 Subject: [PATCH 130/344] =?UTF-8?q?use=20number=20of=20nodes=20with=20expe?= =?UTF-8?q?cted=20debug=20pod=20to=20verify=20if=20daemonset=20is=E2=80=A6?= =?UTF-8?q?=20(#438)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use number of nodes with expected debug pod to verify if daemonset is ready * fix comments --- pkg/config/autodiscover/autodiscover_debug.go | 12 ++++++------ pkg/config/config.go | 8 +++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go index 7c0cdb65b..b0b267ced 100644 --- a/pkg/config/autodiscover/autodiscover_debug.go +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -77,15 +77,15 @@ func DeleteDebugLabel(nodeName string) { // CheckDebugDaemonset checks if the debug pods are deployed properly // the function will try DefaultTimeout/time.Second times -func CheckDebugDaemonset() { +func CheckDebugDaemonset(expectedDebugPods int) { gomega.Eventually(func() bool { log.Debug("check debug daemonset status") - return checkDebugPodsReadiness() + return checkDebugPodsReadiness(expectedDebugPods) }, 60*time.Second, 2*time.Second).Should(gomega.Equal(true)) //nolint: gomnd } // checkDebugPodsReadiness helper function that returns true if the daemonset debug is deployed properly -func checkDebugPodsReadiness() bool { +func checkDebugPodsReadiness(expectedDebugPods int) bool { context := interactive.GetContext(expectersVerboseModeEnabled) tester := ds.NewDaemonSet(DefaultTimeout, debugDaemonSet, defaultNamespace) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) @@ -98,10 +98,10 @@ func checkDebugPodsReadiness() bool { return false } dsStatus := tester.GetStatus() - if dsStatus.Desired == dsStatus.Current && + if expectedDebugPods == dsStatus.Desired && + dsStatus.Desired == dsStatus.Current && + dsStatus.Current == dsStatus.Available && dsStatus.Available == dsStatus.Ready && - dsStatus.Ready == dsStatus.Desired && - dsStatus.Ready != 0 && dsStatus.Misscheduled == 0 { log.Info("daemonset is ready") return true diff --git a/pkg/config/config.go b/pkg/config/config.go index 7207427f1..1c44dabb1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -331,7 +331,13 @@ func (env *TestEnvironment) discoverNodes() { env.NodesUnderTest = env.createNodes(env.Config.Nodes) env.labelNodes() if !autodiscover.IsMinikube() { - autodiscover.CheckDebugDaemonset() + expectedDebugPods := 0 + for _, node := range env.NodesUnderTest { + if node.HasDebugPod() { + expectedDebugPods++ + } + } + autodiscover.CheckDebugDaemonset(expectedDebugPods) autodiscover.FindDebugPods(&env.Config.Partner) for _, debugPod := range env.Config.Partner.ContainersDebugList { env.ContainersToExcludeFromConnectivityTests[debugPod.ContainerIdentifier] = "" From 400867b051156b61d45b7338fedb91131d66ed00 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 3 Nov 2021 12:08:31 -0400 Subject: [PATCH 131/344] add missing suite to readme (#439) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d5a169a01..c98b246de 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,7 @@ Suite|Test Spec Description|Minimum OpenShift Version `networking`|The networking test suite contains tests that check connectivity and networking config related best practices.|4.6.0 `operator`|The operator test suite is designed to test basic Kubernetes Operator functionality.|4.6.0 `platform-alteration`| verifies that key platform configuration is not modified by the CNF under test|4.6.0 - +`observability`| the observability test suite contains tests that check CNF logging is following best practices and that CRDs have status fields|4.6.0 Please consult [CATALOG.md](CATALOG.md) for a detailed description of tests in each suite. From b86786c2542ca6a92e00892a8a9554c29e56a784 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Nov 2021 12:47:06 -0400 Subject: [PATCH 132/344] Bump google.golang.org/grpc from 1.41.0 to 1.42.0 (#437) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.41.0 to 1.42.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.41.0...v1.42.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jun Chen --- go.mod | 6 +----- go.sum | 25 +++++-------------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 1eaec4feb..ee05e5cd6 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,9 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/basgys/goxml2json v1.1.0 github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/golang/mock v1.6.0 github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f - github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057 github.com/onsi/gomega v1.16.0 @@ -20,7 +18,7 @@ require ( github.com/test-network-function/test-network-function-claim v1.0.5 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.42.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -28,7 +26,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect @@ -38,6 +35,5 @@ require ( golang.org/x/text v0.3.6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index e8ea75951..bb76699a1 100644 --- a/go.sum +++ b/go.sum @@ -61,19 +61,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/edcdavid/test-network-function-claim v1.0.6 h1:jb5//83SVvvxII2BJlJJpc3YDHv0L16AoZqGiCrgugI= -github.com/edcdavid/test-network-function-claim v1.0.6/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= -github.com/edcdavid/test-network-function-claim v1.0.7 h1:77GzxaTzyLTL2eqNb4DXnjSXS068+mo4QJg5yZRy1Lk= -github.com/edcdavid/test-network-function-claim v1.0.7/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= -github.com/edcdavid/test-network-function-claim v1.0.8 h1:WgZkLc+jK1IlMFr2w3zxp42+0sRskDMxlONCGfS2ZNg= -github.com/edcdavid/test-network-function-claim v1.0.8/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -85,8 +82,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -161,8 +156,6 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 h1:zHs+jv3LO743/zFGcByu2KmpbliCU2AhjcGgrdTwSG4= -github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -192,7 +185,6 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -224,13 +216,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057 h1:BkX9+SiGBvMou4e1Z+21eTqkqJ8xct8FB+VKzHUTnAY= github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057/go.mod h1:FGGTNz05swxobKgpWKhnxbEiUUxN+CeHRdF9ViWWPDw= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -273,8 +262,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/test-network-function/test-network-function-claim v1.0.4 h1:f/fT8fKHJifVvJ+koaLXbRzH8HG5TKrkwm67tt+dmY8= -github.com/test-network-function/test-network-function-claim v1.0.4/go.mod h1:hDN1y2l8D7K3CX2ZyJThSCBXvVksDLJd1xOMmWqUlaY= github.com/test-network-function/test-network-function-claim v1.0.5 h1:6MSI0hX/6Z5JTHIxg2n+IQF8qBHAKxvZRdwpdMW8eC8= github.com/test-network-function/test-network-function-claim v1.0.5/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -457,7 +444,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -622,8 +608,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -643,7 +629,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From efcad184386f08891f44d62fe6d39ce0199573e4 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 3 Nov 2021 18:07:51 +0100 Subject: [PATCH 133/344] Fix CRD tc identifier test suite. (#441) Co-authored-by: Jun Chen --- CATALOG.md | 16 ++++++++-------- test-network-function/identifiers/identifiers.go | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 1fe7e83cb..845bde9ae 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -68,14 +68,6 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| -### http://test-network-function.com/testcases/diagnostic/crd-status - -Property|Description ----|--- -Version|v1.0.0 -Description|http://test-network-function.com/testcases/diagnostic/crd-status checks that all CRDs have a status subresource specification. -Result Type|informative -Suggested Remediation|make sure that all the CRDs have a meaningful status specification. ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description @@ -172,6 +164,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative Suggested Remediation|Ensure Services are not configured to use NodePort(s). +### http://test-network-function.com/testcases/observability/crd-status + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/observability/crd-status checks that all CRDs have a status subresource specification. +Result Type|informative +Suggested Remediation|make sure that all the CRDs have a meaningful status specification. ### http://test-network-function.com/testcases/operator/install-source Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 22453e303..6949048ce 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -182,6 +182,11 @@ var ( Url: formTestURL(common.ObservabilityTestKey, "container-logging"), Version: versionOne, } + // TestCrdsStatusSubresourceIdentifier ensures all CRDs have a valid status subresource + TestCrdsStatusSubresourceIdentifier = claim.Identifier{ + Url: formTestURL(common.ObservabilityTestKey, "crd-status"), + Version: versionOne, + } // TestShudtownIdentifier ensures pre-stop lifecycle is defined TestShudtownIdentifier = claim.Identifier{ Url: formTestURL(common.LifecycleTestKey, "container-shutdown"), @@ -212,11 +217,6 @@ var ( Url: formTestURL(common.DiagnosticTestKey, "clusterversion"), Version: versionOne, } - // TestCrdsStatusSubresourceIdentifier ensures all CRDs have a valid status subresource - TestCrdsStatusSubresourceIdentifier = claim.Identifier{ - Url: formTestURL(common.DiagnosticTestKey, "crd-status"), - Version: versionOne, - } ) func formDescription(identifier claim.Identifier, description string) string { From 54c10ba7f3aabdbbc090fba49332c157deb0b4ae Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 3 Nov 2021 15:40:34 -0400 Subject: [PATCH 134/344] update logging test (#440) --- CATALOG.md | 8 ++++++++ test-network-function/identifiers/identifiers.go | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/CATALOG.md b/CATALOG.md index 845bde9ae..11a1e09d1 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -164,6 +164,14 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative Suggested Remediation|Ensure Services are not configured to use NodePort(s). +### http://test-network-function.com/testcases/observability/container-logging + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/observability/container-logging check that all containers under test use standard input output and sstandard error when logging +Result Type|informative +Suggested Remediation|make sure containers are not redirecting stdout/stderr ### http://test-network-function.com/testcases/observability/crd-status Property|Description diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 6949048ce..fb03bf42f 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -595,4 +595,12 @@ the changes for you.`, Remediation: `make sure that all the CRDs have a meaningful status specification.`, BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", }, + TestLoggingIdentifier: { + Identifier: TestLoggingIdentifier, + Type: informativeResult, + Description: formDescription(TestLoggingIdentifier, + `check that all containers under test use standard input output and sstandard error when logging`), + Remediation: `make sure containers are not redirecting stdout/stderr`, + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 11.1", + }, } From 93dc5f0ae7486a35fffd46387b59a691c752c8a9 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 3 Nov 2021 17:16:06 -0400 Subject: [PATCH 135/344] platform-alteration-base-image test fix(#443) --- pkg/tnf/handlers/cnffsdiff/cnffsdiff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go index a772292c8..0c2652351 100644 --- a/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go +++ b/pkg/tnf/handlers/cnffsdiff/cnffsdiff.go @@ -96,7 +96,7 @@ func (p *CnfFsDiff) ReelEOF() { // Command returns command line args for checking the fs difference between a container and it's image func Command(containerID string) []string { - return []string{"/host/bin/podman", "diff", "--format", "json", containerID} + return []string{"chroot", "/host", "podman", "diff", "--format", "json", containerID} } // NewFsDiff creates a new `FsDiff` test which checks the fs difference between a container and it's image From 8115e8fd2819a9e44ee89e08ccdaae54173f0023 Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Thu, 4 Nov 2021 14:54:02 +0200 Subject: [PATCH 136/344] update the best practice (#447) --- CATALOG.md | 30 +++++++++++++++++++++++++++++ cmd/tnf/generate/catalog/catalog.go | 1 + 2 files changed, 31 insertions(+) diff --git a/CATALOG.md b/CATALOG.md index 11a1e09d1..bbe6cdb16 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -12,6 +12,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/cluster-role-bindings tests that a Pod does not specify ClusterRoleBindings. Result Type|normative Suggested Remediation|In most cases, Pod's should not have ClusterRoleBindings. The suggested remediation is to remove the need for ClusterRoleBindings, if possible. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.10 and 6.3.6 ### http://test-network-function.com/testcases/access-control/host-resource Property|Description @@ -20,6 +21,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/host-resource tests several aspects of CNF best practices, including: 1. The Pod does not have access to Host Node Networking. 2. The Pod does not have access to Host Node Ports. 3. The Pod cannot access Host Node IPC space. 4. The Pod cannot access Host Node PID space. 5. The Pod is not granted NET_ADMIN SCC. 6. The Pod is not granted SYS_ADMIN SCC. 7. The Pod does not run as root. 8. The Pod does not allow privileged escalation. 9. The Pod is not granted NET_RAW SCC. 10. The Pod is not granted IPC_LOCK SCC. Result Type|normative Suggested Remediation|Ensure that each Pod in the CNF abides by the suggested best practices listed in the test description. In some rare cases, not all best practices can be followed. For example, some CNFs may be required to run as root. Such exceptions should be handled on a case-by-case basis, and should provide a proper justification as to why the best practice(s) cannot be followed. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/namespace Property|Description @@ -28,6 +30,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/namespace tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace. Result Type|normative Suggested Remediation|Ensure that your CNF utilizes a CNF-specific namespace. Additionally, the CNF-specific namespace should not start with "openshift-", except in rare cases. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/access-control/pod-role-bindings Property|Description @@ -36,6 +39,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-role-bindings ensures that a CNF does not utilize RoleBinding(s) in a non-CNF Namespace. Result Type|normative Suggested Remediation|Ensure the CNF is not configured to use RoleBinding(s) in a non-CNF Namespace. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.5 ### http://test-network-function.com/testcases/access-control/pod-service-account Property|Description @@ -44,6 +48,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/access-control/pod-service-account tests that each CNF Pod utilizes a valid Service Account. Result Type|normative Suggested Remediation|Ensure that the each CNF Pod is configured to use a valid Service Account +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.3 and 6.2.7 ### http://test-network-function.com/testcases/affiliated-certification/container-is-certified Property|Description @@ -52,6 +57,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/container-is-certified tests whether container images have passed the Red Hat Container Certification Program (CCP). Result Type|normative Suggested Remediation|Ensure that your container has passed the Red Hat Container Certification Program (CCP). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.7 ### http://test-network-function.com/testcases/affiliated-certification/operator-is-certified Property|Description @@ -60,6 +66,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/affiliated-certification/operator-is-certified tests whether CNF Operators have passed the Red Hat Operator Certification Program (OCP). Result Type|normative Suggested Remediation|Ensure that your Operator has passed Red Hat's Operator Certification Program (OCP). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/diagnostic/clusterversion Property|Description @@ -68,6 +75,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/clusterversion Extracts OCP versions from the cluster. Result Type|informative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/extract-node-information Property|Description @@ -76,6 +84,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/extract-node-information extracts informational information about the cluster. Result Type|informative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.6 ### http://test-network-function.com/testcases/diagnostic/list-cni-plugins Property|Description @@ -84,6 +93,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/list-cni-plugins lists CNI plugins Result Type|normative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.4 and 6.3.7 ### http://test-network-function.com/testcases/diagnostic/nodes-hw-info Property|Description @@ -92,6 +102,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/diagnostic/nodes-hw-info list nodes HW info Result Type|normative Suggested Remediation| +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/container-shutdown Property|Description @@ -100,6 +111,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/container-shutdown Ensure that the containers lifecycle pre-stop management feature is configured. Result Type|normative Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connection). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace period. 3) K8s will clean the remaining processes using KILL signal. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-high-availability Property|Description @@ -108,6 +120,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-high-availability ensures that CNF Pods specify podAntiAffinity rules and replica value is set to more than 1. Result Type|informative Suggested Remediation|In high availability cases, Pod podAntiAffinity rule should be specified for pod scheduling and pod replica value is set to more than 1 . +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-owner-type Property|Description @@ -116,6 +129,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-owner-type tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s). Result Type|normative Suggested Remediation|Deploy the CNF using ReplicaSet/StatefulSet. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.3 and 6.3.8 ### http://test-network-function.com/testcases/lifecycle/pod-recreation Property|Description @@ -124,6 +138,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that a CNF is configured to support High Availability. First, this test cordons and drains a Node that hosts the CNF Pod. Next, the test ensures that OpenShift can re-instantiate the Pod on another Node, and that the actual replica count matches the desired replica count. Result Type|normative Suggested Remediation|Ensure that CNF Pod(s) utilize a configuration that supports High Availability. Additionally, ensure that there are available Nodes in the OpenShift cluster that can be utilized in the event that a host Node fails. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-scheduling Property|Description @@ -132,6 +147,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-scheduling ensures that CNF Pods do not specify nodeSelector or nodeAffinity. In most cases, Pods should allow for instantiation on any underlying Node. Result Type|informative Suggested Remediation|In most cases, Pod's should not specify their host Nodes through nodeSelector or nodeAffinity. However, there are cases in which CNFs require specialized hardware specific to a particular class of Node. As such, this test is purely informative, and will not prevent a CNF from being certified. However, one should have an appropriate justification as to why nodeSelector and/or nodeAffinity is utilized by a CNF. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period Property|Description @@ -140,6 +156,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-termination-grace-period tests whether the terminationGracePeriod is CNF-specific, or if the default (30s) is utilized. This test is informative, and will not affect CNF Certification. In many cases, the default terminationGracePeriod is perfectly acceptable for a CNF. Result Type|informative Suggested Remediation|Choose a terminationGracePeriod that is appropriate for your given CNF. If the default (30s) is appropriate, then feel free to ignore this informative message. This test is meant to raise awareness around how Pods are terminated, and to suggest that a CNF is configured based on its requirements. In addition to a terminationGracePeriod, consider utilizing a termination hook in the case that your application requires special shutdown instructions. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/lifecycle/scaling Property|Description @@ -148,6 +165,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/scaling tests that CNF deployments support scale in/out operations. First, The test starts getting the current replicaCount (N) of the deployment/s with the Pod Under Test. Then, it executes the scale-in oc command for (N-1) replicas. Lastly, it executes the scale-out oc command, restoring the original replicaCount of the deployment/s. Result Type|normative Suggested Remediation|Make sure CNF deployments/replica sets can scale in/out successfully. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/icmpv4-connectivity Property|Description @@ -156,6 +174,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. Result Type|normative Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/networking/service-type Property|Description @@ -164,6 +183,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/networking/service-type tests that each CNF Service does not utilize NodePort(s). Result Type|normative Suggested Remediation|Ensure Services are not configured to use NodePort(s). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.3.1 ### http://test-network-function.com/testcases/observability/container-logging Property|Description @@ -172,6 +192,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/observability/container-logging check that all containers under test use standard input output and sstandard error when logging Result Type|informative Suggested Remediation|make sure containers are not redirecting stdout/stderr +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 11.1 ### http://test-network-function.com/testcases/observability/crd-status Property|Description @@ -180,6 +201,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/observability/crd-status checks that all CRDs have a status subresource specification. Result Type|informative Suggested Remediation|make sure that all the CRDs have a meaningful status specification. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/operator/install-source Property|Description @@ -188,6 +210,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-source tests whether a CNF Operator is installed via OLM. Result Type|normative Suggested Remediation|Ensure that your Operator is installed via OLM. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/operator/install-status Property|Description @@ -196,6 +219,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/operator/install-status Ensures that CNF Operators abide by best practices. The following is tested: 1. The Operator CSV reports "Installed" status. 2. The operator is not installed with privileged rights. Test passes if clusterPermissions is not present in the CSV manifest or is present with no resourceNames under its rules. Result Type|normative Suggested Remediation|Ensure that your Operator abides by the Operator Best Practices mentioned in the description. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.12 and Section 6.3.3 ### http://test-network-function.com/testcases/platform-alteration/base-image Property|Description @@ -204,6 +228,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/base-image ensures that the Container Base Image is not altered post-startup. This test is a heuristic, and ensures that there are no changes to the following directories: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Result Type|normative Suggested Remediation|Ensure that Container applications do not modify the Container Base Image. In particular, ensure that the following directories are not modified: 1) /var/lib/rpm 2) /var/lib/dpkg 3) /bin 4) /sbin 5) /lib 6) /lib64 7) /usr/bin 8) /usr/sbin 9) /usr/lib 10) /usr/lib64 Ensure that all required binaries are built directly into the container image, and are not installed post startup. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.2 ### http://test-network-function.com/testcases/platform-alteration/boot-params Property|Description @@ -212,6 +237,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/boot-params tests that boot parameters are set through the MachineConfigOperator, and not set manually on the Node. Result Type|normative Suggested Remediation|Ensure that boot parameters are set directly through the MachineConfigOperator, or indirectly through the PerformanceAddonOperator. Boot parameters should not be changed directly through the Node, as OpenShift should manage the changes for you. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.13 and 6.2.14 ### http://test-network-function.com/testcases/platform-alteration/hugepages-config Property|Description @@ -220,6 +246,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/isredhat-release Property|Description @@ -228,6 +255,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/isredhat-release verifies if the container base image is redhat. Result Type|normative Suggested Remediation|build a new docker image that's based on UBI (redhat universal base image). +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/sysctl-config Property|Description @@ -236,6 +264,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/lifecycle/pod-recreation tests that no one has changed the node's sysctl configs after the node was created, the tests works by checking if the sysctl configs are consistent with the MachineConfig CR which defines how the node should be configured Result Type|normative Suggested Remediation|You should recreate the node or change the sysctls, recreating is recommended because there might be other unknown changes +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel Property|Description @@ -244,6 +273,7 @@ Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/tainted-node-kernel ensures that the Node(s) hosting CNFs do not utilize tainted kernels. This test case is especially important to support Highly Available CNFs, since when a CNF is re-instantiated on a backup Node, that Node's kernel may not have the same hacks.' Result Type|normative Suggested Remediation|Test failure indicates that the underlying Node's' kernel is tainted. Ensure that you have not altered underlying Node(s) kernels in order to run the CNF. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2.14 ## Test Case Building Blocks Catalog diff --git a/cmd/tnf/generate/catalog/catalog.go b/cmd/tnf/generate/catalog/catalog.go index 602b21e03..bf0ba1f21 100644 --- a/cmd/tnf/generate/catalog/catalog.go +++ b/cmd/tnf/generate/catalog/catalog.go @@ -139,6 +139,7 @@ func outputTestCases() { fmt.Fprintf(os.Stdout, "Description|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Description, "\n", " ")) fmt.Fprintf(os.Stdout, "Result Type|%s\n", identifiers.Catalog[k].Type) fmt.Fprintf(os.Stdout, "Suggested Remediation|%s\n", strings.ReplaceAll(identifiers.Catalog[k].Remediation, "\n", " ")) + fmt.Fprintf(os.Stdout, "Best Practice Reference|%s\n", strings.ReplaceAll(identifiers.Catalog[k].BestPracticeReference, "\n", " ")) } fmt.Println() fmt.Println() From 8a06122f0c70e4e0d59435b82a5d5f5ca0418104 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Thu, 4 Nov 2021 08:22:24 -0500 Subject: [PATCH 137/344] Fix typo in claim/junit filepath check (#445) Co-authored-by: Jun Chen --- pkg/utils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a3c220125..81298947e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -52,12 +52,12 @@ func FilterArray(vs []string, f func(string) bool) []string { func CheckFileExists(filePath, name string) { fullPath, _ := filepath.Abs(filePath) - if _, err := os.Stat(filePath); err == nil { + if _, err := os.Stat(fullPath); err == nil { log.Infof("Path to %s file found and valid: %s ", name, fullPath) } else if errors.Is(err, os.ErrNotExist) { log.Fatalf("Path to %s file not found: %s , Exiting", name, fullPath) } else { - log.Infof("Path to %s file not valid: %s , err=%s, exiting", name, fullPath, err) + log.Fatalf("Path to %s file not valid: %s , err=%s, exiting", name, fullPath, err) } } From 88ffa4a03451bdd85d7ca08f819141169200fa44 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 4 Nov 2021 12:26:30 -0400 Subject: [PATCH 138/344] Skip pods not in Running state in discovery (#442) Co-authored-by: Salaheddine Hamadi --- pkg/config/autodiscover/pod_info.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 81ffd3e0b..86149f828 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -28,6 +28,7 @@ const ( cnfIPsKey = "multusips" cniNetworksStatusKey = "k8s.v1.cni.cncf.io/networks-status" resourceTypePods = "pods" + podPhaseRunning = "Running" ) var ( @@ -58,6 +59,7 @@ type PodResource struct { } `json:"spec"` Status struct { PodIPs []map[string]string `json:"podIPs"` + Phase string `json:"phase"` } `json:"status"` } @@ -163,10 +165,10 @@ func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, err return nil, err } - // Filter out terminating pods + // Filter out terminating pods and pending/unscheduled pods var pods []*PodResource for _, pod := range podList.Items { - if pod.Metadata.DeletionTimestamp == "" { + if pod.Metadata.DeletionTimestamp == "" || pod.Status.Phase != podPhaseRunning { pods = append(pods, pod) } } From d47c9ca5ce870ebb90ec7ed7ae1a0526f01e8732 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Fri, 5 Nov 2021 08:31:12 -0500 Subject: [PATCH 139/344] Fix jq hostPort lookup (#444) --- pkg/tnf/testcases/data/cnf/privilegedpod.go | 4 ++-- pkg/tnf/testcases/files/cnf/privilegedpod.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tnf/testcases/data/cnf/privilegedpod.go b/pkg/tnf/testcases/data/cnf/privilegedpod.go index 8925a60a2..591f9d8eb 100644 --- a/pkg/tnf/testcases/data/cnf/privilegedpod.go +++ b/pkg/tnf/testcases/data/cnf/privilegedpod.go @@ -33,10 +33,10 @@ var PrivilegedPodJSON = string(`{ "name": "HOST_PORT_CHECK", "skiptest": true, "loop": 1, - "command": "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d].ports.hostPort'", + "command": "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d] | select(.ports) | .ports[].hostPort'", "action": "allow", "expectedstatus": [ - "NULL_FALSE" + "^()*$" ] }, { diff --git a/pkg/tnf/testcases/files/cnf/privilegedpod.yml b/pkg/tnf/testcases/files/cnf/privilegedpod.yml index 56502649a..dbe154706 100644 --- a/pkg/tnf/testcases/files/cnf/privilegedpod.yml +++ b/pkg/tnf/testcases/files/cnf/privilegedpod.yml @@ -10,11 +10,11 @@ testcase: - name: HOST_PORT_CHECK skiptest: true loop: 0 - command: "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d].ports.hostPort'" + command: "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d] | select(.ports) | .ports[].hostPort'" action: allow expectedType: "regex" expectedstatus: - - NULL_FALSE + - "^()*$" - name: HOST_PATH_CHECK skiptest: true loop: 0 From 75ca60356dac8ac6c0b1639d538b776e018f474a Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 5 Nov 2021 09:46:48 -0400 Subject: [PATCH 140/344] Fix error handling in platform tests (#451) * Fix error handling in platform tests * Lint fix * Addressing review comments --- pkg/tnf/test.go | 22 ++++++++----- test-network-function/platform/suite.go | 41 +++++++++++++++++-------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index c73d22514..1f424f7bf 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -135,15 +135,23 @@ func (t *Test) RunAndValidateWithFailureCallback(cb func()) { gomega.Expect(err).To(gomega.BeNil()) } -// RunWithFailureCallback runs the test, invokes the cb on failure +// RunWithCallbacks runs the test, invokes the cb on failure/error/success // This is useful when the testcase needs to continue whether this test result is success or not -func (t *Test) RunWithFailureCallback(cb func()) { +func (t *Test) RunWithCallbacks(successCb, failureCb func(), errorCb func(error)) { testResult, err := t.Run() - if testResult == FAILURE && cb != nil { - cb() - } - if err != nil { - log.Warnf("Test %s error: %v", t.tester.GetIdentifier().URL, err) + switch testResult { + case SUCCESS: + if successCb != nil { + successCb() + } + case FAILURE: + if failureCb != nil { + failureCb() + } + case ERROR: + if errorCb != nil { + errorCb(err) + } } } diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index b61b20a28..2b9f6d614 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -186,6 +186,7 @@ func testContainersFsDiff(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestUnalteredBaseImageIdentifier) ginkgo.It(testID, func() { var badContainers []string + var errContainers []string for _, cut := range env.ContainersUnderTest { podName := cut.Oc.GetPodName() containerName := cut.Oc.GetPodContainerName() @@ -194,12 +195,21 @@ func testContainersFsDiff(env *config.TestEnvironment) { ginkgo.By(fmt.Sprintf("%s(%s) should not install new packages after starting", podName, containerName)) nodeOc := env.NodesUnderTest[nodeName].Oc test := newContainerFsDiffTest(nodeName, nodeOc, containerOC) - test.RunWithFailureCallback(func() { + var message string + test.RunWithCallbacks(nil, func() { badContainers = append(badContainers, containerName) - ginkgo.By(fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName)) + message = fmt.Sprintf("pod %s container %s did update/install/modify additional packages", podName, containerName) + }, func(err error) { + errContainers = append(errContainers, containerName) + message = fmt.Sprintf("Failed to check pod %s container %s for additional packages due to: %v", podName, containerName, err) }) + _, err := ginkgo.GinkgoWriter.Write([]byte(message)) + if err != nil { + log.Errorf("Ginkgo writer could not write because: %s", err) + } } gomega.Expect(badContainers).To(gomega.BeNil()) + gomega.Expect(errContainers).To(gomega.BeNil()) }) }) } @@ -385,6 +395,7 @@ func testTainted(env *config.TestEnvironment) { ginkgo.By("Testing tainted nodes in cluster") var taintedNodes []string + var errNodes []string for _, node := range env.NodesUnderTest { if !node.HasDebugPod() { continue @@ -393,24 +404,30 @@ func testTainted(env *config.TestEnvironment) { tester := nodetainted.NewNodeTainted(common.DefaultTimeout) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - test.RunWithFailureCallback(func() { + var message string + test.RunWithCallbacks(func() { + message = fmt.Sprintf("Decoded tainted kernel causes (code=0) for node %s : None\n", node.Name) + }, func() { + var taintedBitmap uint64 + taintedBitmap, err = strconv.ParseUint(tester.Match, 10, 32) //nolint:gomnd // base 10 and uint32 + if err != nil { + message = fmt.Sprintf("Could not decode tainted kernel causes (code=%d) for node %s\n", taintedBitmap, node.Name) + } else { + message = fmt.Sprintf("Decoded tainted kernel causes (code=%d) for node %s : %s\n", taintedBitmap, node.Name, printTainted(taintedBitmap)) + } taintedNodes = append(taintedNodes, node.Name) + }, func(e error) { + message = fmt.Sprintf("Failed to retrieve tainted kernel code for node %s\n", node.Name) + errNodes = append(errNodes, node.Name) }) - taintedBitmap, err := strconv.ParseUint(tester.Match, 10, 32) //nolint:gomnd // base 10 and uint32 - var message string - if err != nil { - message = fmt.Sprintf("Could not decode tainted kernel causes (code=%d) for node %s\n", taintedBitmap, node.Name) - } else if taintedBitmap != 0 { - message = fmt.Sprintf("Decoded tainted kernel causes (code=%d) for node %s : %s\n", taintedBitmap, node.Name, printTainted(taintedBitmap)) - } else { - message = fmt.Sprintf("Decoded tainted kernel causes (code=%d) for node %s : None\n", taintedBitmap, node.Name) - } + _, err = ginkgo.GinkgoWriter.Write([]byte(message)) if err != nil { log.Errorf("Ginkgo writer could not write because: %s", err) } } gomega.Expect(taintedNodes).To(gomega.BeNil()) + gomega.Expect(errNodes).To(gomega.BeNil()) }) } From 26f4fca0151cc770a41a75e09ddec4a8a22d4e53 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Fri, 5 Nov 2021 09:05:25 -0500 Subject: [PATCH 141/344] More typo cleanup (#453) Co-authored-by: Jun Chen --- CATALOG.md | 6 +++--- DEVELOPING.md | 6 +++--- README.md | 6 +++--- pkg/config/config.go | 2 +- pkg/config/configsections/common.go | 2 +- pkg/tnf/identifier/identifiers.go | 2 +- test-network-function/identifiers/identifiers.go | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index bbe6cdb16..0b4988148 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -189,7 +189,7 @@ Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/site Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/observability/container-logging check that all containers under test use standard input output and sstandard error when logging +Description|http://test-network-function.com/testcases/observability/container-logging check that all containers under test use standard input output and standard error when logging Result Type|informative Suggested Remediation|make sure containers are not redirecting stdout/stderr Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 11.1 @@ -245,7 +245,7 @@ Property|Description Version|v1.0.0 Description|http://test-network-function.com/testcases/platform-alteration/hugepages-config checks to see that HugePage settings have been configured through MachineConfig, and not manually on the underlying Node. This test case applies only to Nodes that are configured with the "worker" MachineConfigSet. First, the "worker" MachineConfig is polled, and the Hugepage settings are extracted. Next, the underlying Nodes are polled for configured HugePages through inspection of /proc/meminfo. The results are compared, and the test passes only if they are the same. Result Type|normative -Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. +Suggested Remediation|HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the PerformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings. Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 ### http://test-network-function.com/testcases/platform-alteration/isredhat-release @@ -332,7 +332,7 @@ Runtime Binaries Required|`oc`, `jq` Property|Description ---|--- Version|v1.0.0 -Description|extracts the csi driver info in the cluser +Description|extracts the csi driver info in the cluster Result Type|normative Intrusive|false Modifications Persist After Test|false diff --git a/DEVELOPING.md b/DEVELOPING.md index c6650454f..e93e03cab 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -529,7 +529,7 @@ called once. The logic for determining the test result is up to the test writer. This particular implementation analyzes the match output to determine the result. -1) If the provided `destination` results in an `Indvalid Argument`, then `tnf.ERROR` is returned. +1) If the provided `destination` results in an `Invalid Argument`, then `tnf.ERROR` is returned. 2) If the ping summary regular expression matched, then: * `tnf.ERROR` if there were PING transmit errors * `tnf.SUCCESS` if a maximum of a single packet was lost @@ -581,7 +581,7 @@ of such PTY implementations can be found in [examples/pty](./examples/pty). The current tests frequently use `jq` to process structured output from `oc -o json`. `oc` also allows use of [Go Templates](https://www.openshift.com/blog/customizing-oc-output-with-go-templates) for processing structured output. -This is potentially more powerful than using `jq` as it allows building highly customised output of multiple resources +This is potentially more powerful than using `jq` as it allows building highly customized output of multiple resources simultaneously without adding dependencies. Conversely `jq` is widely available and commonly used, and has been sufficient for all cases so far. It is up to the author of a contribution to decide which approach is best suited to the task at hand. @@ -622,7 +622,7 @@ The command relays on golang templates located in [pkg/tnf/handlers/handler_temp The result of each test execution is included in the claim file. Sometimes it is convenient to add informational messages regarding the test execution. In order to add informational messages to your test use the function `ginkgo.GinkgoWriter`. -This function adds an additional message that will appear in the `CapturedTestOutput` section of the claim file, together witht the output of the by directives. +This function adds an additional message that will appear in the `CapturedTestOutput` section of the claim file, together with the output of the by directives. Each added message will be written to claim file even if test failed or error occurred in the middle of the test. Example usage: diff --git a/README.md b/README.md index c98b246de..5479ea2d9 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The autodiscovery mechanism will create a list of all CRD names in the cluster w ### testTarget #### podsUnderTest / containersUnderTest -This section is usually not required if labels defined in the section above cover all resources that should be tested. If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, instrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. The pods and containers explicitly configured here are added to the target pod/container lists populated through label matching. +This section is usually not required if labels defined in the section above cover all resources that should be tested. If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, intrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. The pods and containers explicitly configured here are added to the target pod/container lists populated through label matching. For both configured and discovered pods/containers, the autodiscovery mechanism will attempt to identify the default network device and all the IP addresses of the pods it needs for network connectivity tests, though that information can be explicitly set using annotations if needed. For Pod IPs: @@ -76,7 +76,7 @@ be seen in [cnf-certification-test-partner](https://github.com/test-network-func the first entry found with `"default"=true` is used. This annotation is automatically managed in OpenShift but may not be present in K8s. -If multus IP addresses are dicovered or configured, the partner pod needs to be deployed in the same namespace as the multus network interface for the connectivity test to pass. Refer to instruction [here](#specify-the-target-namespace-for-partner-pod-deployment). +If multus IP addresses are discovered or configured, the partner pod needs to be deployed in the same namespace as the multus network interface for the connectivity test to pass. Refer to instruction [here](#specify-the-target-namespace-for-partner-pod-deployment). If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. `ping`), it should be given the label `test-network-function.com/skip_connectivity_tests` to exclude it from those tests. The label value is @@ -94,7 +94,7 @@ the subscription for this CSV. If unset, the CSV name will be used. ### testPartner -This section can also be discovered automatically and should be left commented out unless the parter pods are modified from the original version in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/local-test-infra/) +This section can also be discovered automatically and should be left commented out unless the partner pods are modified from the original version in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/local-test-infra/) ### certifiedcontainerinfo and certifiedoperatorinfo diff --git a/pkg/config/config.go b/pkg/config/config.go index 1c44dabb1..d5f723daa 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -143,7 +143,7 @@ func getContainerDefaultNetworkIPAddress(oc *interactive.Oc, dev string) (string oc.GetPodName(), oc.GetPodContainerName(), oc.GetPodNamespace(), result, err) } -// TestEnvironment includes the representation of the current state of the test targets and parters as well as the test configuration +// TestEnvironment includes the representation of the current state of the test targets and partners as well as the test configuration type TestEnvironment struct { ContainersUnderTest map[configsections.ContainerIdentifier]*Container PartnerContainers map[configsections.ContainerIdentifier]*Container diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index ad9882aa4..7f757c2a8 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -66,7 +66,7 @@ type TestConfiguration struct { // TestPartner contains the helper containers that can be used to facilitate tests type TestPartner struct { - // ContainerConfigList is the list parter containers that facilitates tests + // ContainerConfigList is the list partner containers that facilitates tests ContainerConfigList []ContainerConfig `yaml:"partnerContainers" json:"partnerContainers"` // TestOrchestratorID is the id of the partner container for conducting connectivity tests TestOrchestratorID ContainerIdentifier `yaml:"testOrchestrator" json:"testOrchestrator"` diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index e051908ef..1081ec061 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -601,7 +601,7 @@ var Catalog = map[string]TestCatalogEntry{ }, csiDriverIdentifierURL: { Identifier: CSIDriverIdentifier, - Description: "extracts the csi driver info in the cluser", + Description: "extracts the csi driver info in the cluster", Type: Normative, IntrusionSettings: IntrusionSettings{ ModifiesSystem: false, diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index fb03bf42f..80158e575 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -290,7 +290,7 @@ cannot be followed.`, Identifier: TestHugepagesNotManuallyManipulated, Type: normativeResult, Remediation: `HugePage settings should be configured either directly through the MachineConfigOperator or indirectly using the -PeformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can +PerformanceAddonOperator. This ensures that OpenShift is aware of the special MachineConfig requirements, and can provision your CNF on a Node that is part of the corresponding MachineConfigSet. Avoid making changes directly to an underlying Node, and let OpenShift handle the heavy lifting of configuring advanced settings.`, Description: formDescription(TestHugepagesNotManuallyManipulated, @@ -599,7 +599,7 @@ the changes for you.`, Identifier: TestLoggingIdentifier, Type: informativeResult, Description: formDescription(TestLoggingIdentifier, - `check that all containers under test use standard input output and sstandard error when logging`), + `check that all containers under test use standard input output and standard error when logging`), Remediation: `make sure containers are not redirecting stdout/stderr`, BestPracticeReference: bestPracticeDocV1dot2URL + " Section 11.1", }, From 27f65a8fc36ef700147944bce7705ad4b18f9444 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Fri, 5 Nov 2021 09:54:19 -0500 Subject: [PATCH 142/344] Go 1.17.3 update (#452) Co-authored-by: Jun Chen --- .github/workflows/merge.yaml | 2 +- .github/workflows/pre-main.yaml | 6 +++--- Dockerfile | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml index 7b095f2cd..d9b327f7d 100644 --- a/.github/workflows/merge.yaml +++ b/.github/workflows/merge.yaml @@ -15,7 +15,7 @@ jobs: - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: 1.17.2 + go-version: 1.17.3 - name: Check out code into the Go module directory uses: actions/checkout@v2 diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 10e95da02..c05b10da8 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -32,7 +32,7 @@ jobs: - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: 1.17.2 + go-version: 1.17.3 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -80,7 +80,7 @@ jobs: - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: 1.17.2 + go-version: 1.17.3 - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -104,7 +104,7 @@ jobs: - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: 1.17.2 + go-version: 1.17.3 - name: Check out code into the Go module directory uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index d6797658b..b39c6783f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN yum install -y gcc git jq make wget # Install Go binary ENV GO_DL_URL="https://golang.org/dl" -ENV GO_BIN_TAR="go1.17.2.linux-amd64.tar.gz" +ENV GO_BIN_TAR="go1.17.3.linux-amd64.tar.gz" ENV GO_BIN_URL_x86_64=${GO_DL_URL}/${GO_BIN_TAR} ENV GOPATH="/root/go" RUN if [[ "$(uname -m)" -eq "x86_64" ]] ; then \ From a4d51713e1c4e331ce4e799f4100bdef4bb28aa7 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 5 Nov 2021 11:23:02 -0500 Subject: [PATCH 143/344] exclude default pod IP from multus ip list (#436) * exclude default pod IP from multus ip list * Fix makefile target dependency (#435) * Fix makefile target dependency * Remove outdated targets Co-authored-by: Jun Chen Co-authored-by: Salaheddine Hamadi --- pkg/config/autodiscover/container_test.go | 4 +--- pkg/config/autodiscover/pod_info.go | 10 ++++++---- test-network-function/networking/suite.go | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/config/autodiscover/container_test.go b/pkg/config/autodiscover/container_test.go index 977ed305c..9a41bb510 100644 --- a/pkg/config/autodiscover/container_test.go +++ b/pkg/config/autodiscover/container_test.go @@ -41,9 +41,7 @@ func TestBuildContainersFromPodResource(t *testing.T) { assert.Equal(t, "eth1", subjectContainers[0].DefaultNetworkDevice) // Check correct IPs are chosen - assert.Equal(t, 1, len(orchestratorContainers[0].MultusIPAddresses)) - assert.Equal(t, "1.1.1.1", orchestratorContainers[0].MultusIPAddresses[0]) - assert.NotEqual(t, "2.2.2.2", orchestratorContainers[0].MultusIPAddresses[0]) + assert.Equal(t, 0, len(orchestratorContainers[0].MultusIPAddresses)) // test-network-function.com/multusips should be used for the test subject container. assert.Equal(t, 2, len(subjectContainers[0].MultusIPAddresses)) assert.Equal(t, "3.3.3.3", subjectContainers[0].MultusIPAddresses[0]) diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 86149f828..95cbe4545 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -134,15 +134,17 @@ func (pr *PodResource) getPodIPs() (ips []string, err error) { if err != nil { return nil, pr.annotationUnmarshalError(cniNetworksStatusKey, err) } + // If this is the default interface, skip it as it is tested separately + // Otherwise add all non default interfaces for _, cniInterface := range cniInfo { - ips = append(ips, cniInterface.IPs...) + if !cniInterface.Default { + ips = append(ips, cniInterface.IPs...) + } } return } log.Warn("Could not establish pod IPs from annotations, please manually set the 'test-network-function.com/multusips' annotation for complete test coverage") - for _, ip := range pr.Status.PodIPs { - ips = append(ips, ip["ip"]) - } + return } diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index eeb4c664a..3f34f42f8 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -110,10 +110,16 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { if env.TestOrchestrator == nil { ginkgo.Skip("Orchestrator is not deployed, skip this test") } + found := false for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { continue } + found = true + if len(cut.ContainerConfiguration.MultusIPAddresses) == 0 { + ginkgo.Skip("No Multus IPs detected") + } + for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), @@ -121,6 +127,9 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { cut.DefaultNetworkIPAddress)) testPing(testOrchestrator.Oc, multusIPAddress, count) } + if !found { + ginkgo.Skip("No container found suitable for Multus connectivity test") + } } }) }) From 71c20d0958ef598e93dbdfc99c6345a9b9e0b0b7 Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Fri, 5 Nov 2021 19:59:54 +0200 Subject: [PATCH 144/344] Add IPC_LOCK and NET_RAW to pod SCC check (#449) * Add SCC capability checks for IPC_LOCK and NET_RAW * fixed make lint * Updated the file privilegedpod.yml * fixed make lint * Resolved comment of Jun * Updated file privilegefpod.go Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen --- pkg/tnf/testcases/data/cnf/privilegedpod.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/tnf/testcases/data/cnf/privilegedpod.go b/pkg/tnf/testcases/data/cnf/privilegedpod.go index 591f9d8eb..4942a4830 100644 --- a/pkg/tnf/testcases/data/cnf/privilegedpod.go +++ b/pkg/tnf/testcases/data/cnf/privilegedpod.go @@ -78,7 +78,9 @@ var PrivilegedPodJSON = string(`{ "action": "deny", "expectedstatus": [ "NET_ADMIN", - "SYS_ADMIN" + "SYS_ADMIN", + "NET_RAW", + "IPC_LOCK" ] }, { From 031ad119a617bd6e13a120e2f5aeea971051b607 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 5 Nov 2021 18:31:57 -0400 Subject: [PATCH 145/344] Fix deployment discovery with non qualified taget pod label names (#454) --- pkg/config/autodiscover/autodiscover.go | 6 +++--- pkg/config/autodiscover/deployment_info.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 481b0ccdf..eb3fd398f 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -48,11 +48,11 @@ func PerformAutoDiscovery() (doAuto bool) { return !doAuto } -func buildLabelName(labelNS, labelName string) string { - if labelNS == "" { +func buildLabelName(labelPrefix, labelName string) string { + if labelPrefix == "" { return labelName } - return fmt.Sprintf(labelTemplate, labelNS, labelName) + return fmt.Sprintf(labelTemplate, labelPrefix, labelName) } func buildAnnotationName(annotationName string) string { diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index 03795089c..6c09a3810 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -80,7 +80,7 @@ func (deployment *DeploymentResource) GetLabels() map[string]string { // GetTargetDeploymentsByNamespace will return all deployments that have pods with a given label. func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsections.Label) (*DeploymentList, error) { - labelQuery := fmt.Sprintf("\"%s/%s\"==\"%s\"", targetLabel.Prefix, targetLabel.Name, targetLabel.Value) + labelQuery := fmt.Sprintf("\"%s\"==\"%s\"", buildLabelName(targetLabel.Prefix, targetLabel.Name), targetLabel.Value) jqArgs := fmt.Sprintf("'[.items[] | select(.spec.template.metadata.labels.%s)]'", labelQuery) ocCmd := fmt.Sprintf("oc get %s -n %s -o json | jq %s", resourceTypeDeployment, namespace, jqArgs) From e2d163adfde55566700fc6fb7978613b87f2e7e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 11:26:20 -0500 Subject: [PATCH 146/344] Bump github.com/onsi/gomega from 1.16.0 to 1.17.0 (#456) Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ee05e5cd6..6daf619a3 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f github.com/kr/pretty v0.2.1 // indirect github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057 - github.com/onsi/gomega v1.16.0 + github.com/onsi/gomega v1.17.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index bb76699a1..46fc2bee5 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/onsi/ginkgo v1.16.6-0.20211014152641-f228134fe057/go.mod h1:FGGTNz05s github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= -github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 85dfa3bc729316368f9b519d45be461ce703a7ba Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 8 Nov 2021 14:45:26 -0600 Subject: [PATCH 147/344] Lifecycle suite issues (#446) * Fix fatal exit * PR review Jun * PR Gonzalo Co-authored-by: Jun Chen --- pkg/config/config.go | 37 +++++++++++++++++++----- pkg/tnf/interactive/oc.go | 5 ++++ pkg/tnf/interactive/spawner.go | 5 ++++ test-network-function/common/suite.go | 1 + test-network-function/lifecycle/suite.go | 31 +++++++++++--------- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index d5f723daa..b9f617c2d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -111,11 +111,12 @@ func getOcSession(pod, container, namespace string, timeout time.Duration, optio gomega.Expect(err).To(gomega.BeNil()) // Set up a go routine which reads from the error channel go func() { + log.Debugf("start watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) select { case err := <-outCh: log.Fatalf("OC session to container %s/%s is broken due to: %v, aborting the test run", oc.GetPodName(), oc.GetPodContainerName(), err) case <-oc.GetDoneChannel(): - log.Infof("stop watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) + log.Debugf("stop watching the session with container %s/%s", oc.GetPodName(), oc.GetPodContainerName()) } }() ocChan <- oc @@ -202,25 +203,46 @@ func (env *TestEnvironment) LoadAndRefresh() { } } +// Resets the environment during the drain test since all the connections are affected func (env *TestEnvironment) reset() { log.Debug("clean up environment Test structure") + env.ResetOc() env.Config.Partner = configsections.TestPartner{} env.Config.TestTarget = configsections.TestTarget{} env.TestOrchestrator = nil // Delete Oc debug sessions before re-creating them for name, node := range env.NodesUnderTest { if node.HasDebugPod() { - node.Oc.Close() autodiscover.DeleteDebugLabel(name) } } + env.NodesUnderTest = nil + env.Config.Nodes = nil + env.DebugContainers = nil +} + +// Resets the environment during the intrusive tests since all the connections are affected +func (env *TestEnvironment) ResetOc() { + log.Debug("Reset Oc sessions") + // Delete Oc debug sessions before re-creating them + for _, node := range env.NodesUnderTest { + if node.HasDebugPod() { + log.Infof("Closing session to node %s", node.Name) + node.Oc.Close() + node.Oc = nil + } + } // Delete all remaining sessions before re-creating them for _, cut := range env.ContainersUnderTest { cut.Oc.Close() + cut.Oc = nil + } + + // Delete all remaining partner sessions before re-creating them + for _, cut := range env.PartnerContainers { + cut.Oc.Close() + cut.Oc = nil } - env.NodesUnderTest = nil - env.Config.Nodes = nil - env.DebugContainers = nil } func (env *TestEnvironment) doAutodiscover() { @@ -275,7 +297,7 @@ func (env *TestEnvironment) labelNodes() { if node.IsWorker() && workerNode == "" { workerNode = name } - if node.IsMaster() && node.HasDebugPod() { + if node.IsWorker() && node.HasDebugPod() { workerNode = "" break } @@ -300,7 +322,7 @@ func (env *TestEnvironment) createNodes(nodes map[string]configsections.Node) ma defer log.Debug("autodiscovery: create nodes done") nodesConfig := make(map[string]*NodeConfig) for _, n := range nodes { - nodesConfig[n.Name] = &NodeConfig{Node: n, Name: n.Name, Oc: nil, deployment: false} + nodesConfig[n.Name] = &NodeConfig{Node: n, Name: n.Name, Oc: nil, deployment: false, debug: false} } for _, c := range env.ContainersUnderTest { nodeName := c.ContainerConfiguration.NodeName @@ -330,6 +352,7 @@ func (env *TestEnvironment) AttachDebugPodsToNodes() { func (env *TestEnvironment) discoverNodes() { env.NodesUnderTest = env.createNodes(env.Config.Nodes) env.labelNodes() + if !autodiscover.IsMinikube() { expectedDebugPods := 0 for _, node := range env.NodesUnderTest { diff --git a/pkg/tnf/interactive/oc.go b/pkg/tnf/interactive/oc.go index 9b83e6e62..f98ff5753 100644 --- a/pkg/tnf/interactive/oc.go +++ b/pkg/tnf/interactive/oc.go @@ -125,4 +125,9 @@ func (o *Oc) Close() { } log.Debugf("send close to channel pod %s/%s ", o.pod, o.container) o.doneChannel <- true + close(o.doneChannel) + err := (*(o.expecter)).Close() + if err != nil { + log.Errorf("Oc session close failed because of: %s", err) + } } diff --git a/pkg/tnf/interactive/spawner.go b/pkg/tnf/interactive/spawner.go index c1dddcbf3..61891d948 100644 --- a/pkg/tnf/interactive/spawner.go +++ b/pkg/tnf/interactive/spawner.go @@ -122,6 +122,11 @@ func (e *ExecSpawnFunc) StderrPipe() (io.Reader, error) { return e.cmd.StderrPipe() } +// Wait wraps exec.Cmd.Wait. +func (e *ExecSpawnFunc) Close() error { + return e.cmd.Process.Kill() +} + // Spawner provides an interface for creating interactive sessions such as oc, ssh, or shell. type Spawner interface { // Spawn creates the interactive session. diff --git a/test-network-function/common/suite.go b/test-network-function/common/suite.go index fdbc4e545..3c808f554 100644 --- a/test-network-function/common/suite.go +++ b/test-network-function/common/suite.go @@ -41,6 +41,7 @@ var _ = ginkgo.AfterSuite(func() { continue } node.Oc.Close() + node.Oc = nil autodiscover.DeleteDebugLabel(name) } }) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 78f249bf8..28c11ae91 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -113,9 +113,10 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { } }) -func waitForAllDeploymentsReady(namespace string, timeout, pollingPeriod time.Duration) { +func waitForAllDeploymentsReady(namespace string, timeout, pollingPeriod time.Duration) { //nolint:unparam // it is fine to use always the same value for timeout gomega.Eventually(func() []string { _, notReadyDeployments := getDeployments(namespace) + log.Debugf("Waiting for deployments to get ready, remaining: %d deployments", len(notReadyDeployments)) return notReadyDeployments }, timeout, pollingPeriod).Should(gomega.HaveLen(0)) } @@ -149,16 +150,7 @@ func closeOcSessionsByDeployment(containers map[configsections.ContainerIdentifi if cid.Namespace == deployment.Namespace && strings.HasPrefix(cid.PodName, deployment.Name+"-") { log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) c.Oc.Close() - delete(containers, cid) - } - } -} - -func closeOcSessionsByNode(containers map[configsections.ContainerIdentifier]*config.Container, nodeName string) { - for cid, c := range containers { - if cid.NodeName == nodeName { - log.Infof("Closing session to %s %s", cid.PodName, cid.ContainerName) - c.Oc.Close() + c.Oc = nil delete(containers, cid) } } @@ -309,8 +301,10 @@ func testPodsRecreation(env *config.TestEnvironment) { log.Debug("node ", n.Name, " has no deployment, skip draining") continue } - closeOcSessionsByNode(env.ContainersUnderTest, n.Name) - closeOcSessionsByNode(env.PartnerContainers, n.Name) + // We need to delete all Oc sessions because the drain operation is often deleting oauth-openshift pod + // This result in lost connectivity oc sessions + env.ResetOc() + // drain node drainNode(n.Name) // should go in this @@ -324,6 +318,9 @@ func testPodsRecreation(env *config.TestEnvironment) { } uncordonNode(n.Name) + + // wait for all deployment to be ready otherwise, pods might be unreacheable during the next discovery + waitForAllDeploymentsReady(env.NameSpaceUnderTest, scalingTimeout, scalingPollingPeriod) } }) } @@ -341,6 +338,9 @@ func getDeployments(namespace string) (deployments dp.DeploymentMap, notReadyDep for name, d := range deployments { if d.Unavailable != 0 || d.Ready != d.Replicas || d.Available != d.Replicas || d.UpToDate != d.Replicas { notReadyDeployments = append(notReadyDeployments, name) + log.Tracef("deployment %s: not ready", name) + } else { + log.Tracef("deployment %s: ready", name) } } @@ -352,7 +352,10 @@ func drainNode(node string) { tester := dd.NewDeploymentsDrain(drainTimeout, node) test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidate() + result, err := test.Run() + if err != nil || result == tnf.ERROR { + log.Fatalf("Test skipped because of draining node failure - platform issue") + } } func uncordonNode(node string) { From e4d73e56695ccd880f17ef9cd32e911b4609e46a Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 10 Nov 2021 07:52:29 -0600 Subject: [PATCH 148/344] Fix rolebinding result parsing (#457) --- pkg/tnf/handlers/rolebinding/rolebinding.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tnf/handlers/rolebinding/rolebinding.go b/pkg/tnf/handlers/rolebinding/rolebinding.go index 6f9002b53..9719f7b4c 100644 --- a/pkg/tnf/handlers/rolebinding/rolebinding.go +++ b/pkg/tnf/handlers/rolebinding/rolebinding.go @@ -40,7 +40,7 @@ type RoleBinding struct { // NewRoleBinding creates a new RoleBinding tnf.Test. func NewRoleBinding(timeout time.Duration, serviceAccountName, podNamespace string) *RoleBinding { - serviceAccountSubString := "name:" + serviceAccountName + " namespace:" + podNamespace + serviceAccountSubString := "name:\\b" + serviceAccountName + "\\b namespace:\\b" + podNamespace + "\\b" return &RoleBinding{ podNamespace: podNamespace, timeout: timeout, From 85337e4a189151de42393d1dc8af01ea7c0a8050 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Wed, 10 Nov 2021 19:16:58 -0500 Subject: [PATCH 149/344] add multinamespaces support (#455) * add multinamespaces support * Update pkg/config/autodiscover/autodiscover_targets.go Co-authored-by: Brandon Palm * fix PR's review * rebase and fix build issue * fix lint issues * reset namespaces in reset * try workflow fix * Revert "try workflow fix" This reverts commit 7e680a90563dbb362dbeeb956f7fd5fe9be6e741. Co-authored-by: Brandon Palm Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- .../autodiscover/autodiscover_targets.go | 48 ++++++++++--------- pkg/config/config.go | 14 +++--- pkg/tnf/handlers/deployments/deployments.go | 11 +++-- .../handlers/deployments/deployments_test.go | 8 ++-- test-network-function/lifecycle/suite.go | 46 ++++++++++-------- test-network-function/networking/suite.go | 12 +++-- 6 files changed, 78 insertions(+), 61 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 7046fc9d5..c66d292db 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -46,37 +46,39 @@ var ( // FindTestTarget finds test targets from the current state of the cluster, // using labels and annotations, and add them to the `configsections.TestTarget` passed in. -func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget, namespace string) { - // find pods by label - for _, l := range labels { - pods, err := GetPodsByLabel(l, namespace) - if err == nil { - for i := range pods.Items { - target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(pods.Items[i])) - target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(pods.Items[i])...) +func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget, namespaces []string) { + for _, ns := range namespaces { + // find pods by label + for _, l := range labels { + pods, err := GetPodsByLabel(l, ns) + if err == nil { + for i := range pods.Items { + target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(pods.Items[i])) + target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(pods.Items[i])...) + } + } else { + log.Warnf("failed to query by label: %v %v", l, err) } - } else { - log.Warnf("failed to query by label: %v %v", l, err) } - } - // Containers to exclude from connectivity tests are optional - identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}, namespace) - target.ExcludeContainersFromConnectivityTests = identifiers + // Containers to exclude from connectivity tests are optional + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}, ns) + target.ExcludeContainersFromConnectivityTests = append(target.ExcludeContainersFromConnectivityTests, identifiers...) - if err != nil { - log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) - } + if err != nil { + log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) + } + + csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue, ns) + if err != nil { + log.Warnf("an error (%s) occurred when looking for operators by label", err) + } - csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue, namespace) - if err == nil { for i := range csvs.Items { target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csvs.Items[i])) } - } else { - log.Warnf("an error (%s) occurred when looking for operaters by label", err) - } - target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target, namespace)...) + target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target, ns)...) + } target.Nodes = GetNodesList() } diff --git a/pkg/config/config.go b/pkg/config/config.go index b9f617c2d..4f94a703c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,6 +36,7 @@ const ( configurationFilePathEnvironmentVariableKey = "TNF_CONFIGURATION_PATH" defaultConfigurationFilePath = "tnf_config.yml" defaultTimeoutSeconds = 10 + defaultNamespace = "default" ) var ( @@ -152,7 +153,7 @@ type TestEnvironment struct { PodsUnderTest []configsections.Pod DeploymentsUnderTest []configsections.Deployment OperatorsUnderTest []configsections.Operator - NameSpaceUnderTest string + NameSpacesUnderTest []string CrdNames []string NodesUnderTest map[string]*NodeConfig @@ -216,6 +217,7 @@ func (env *TestEnvironment) reset() { autodiscover.DeleteDebugLabel(name) } } + env.NameSpacesUnderTest = nil env.NodesUnderTest = nil env.Config.Nodes = nil env.DebugContainers = nil @@ -247,12 +249,12 @@ func (env *TestEnvironment) ResetOc() { func (env *TestEnvironment) doAutodiscover() { log.Debug("start auto discovery") - if len(env.Config.TargetNameSpaces) != 1 { - log.Fatal("a single namespace should be specified in config file") + for _, ns := range env.Config.TargetNameSpaces { + env.NameSpacesUnderTest = append(env.NameSpacesUnderTest, ns.Name) } - env.NameSpaceUnderTest = env.Config.TargetNameSpaces[0].Name + if autodiscover.PerformAutoDiscovery() { - autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget, env.NameSpaceUnderTest) + autodiscover.FindTestTarget(env.Config.TargetPodLabels, &env.Config.TestTarget, env.NameSpacesUnderTest) } env.ContainersToExcludeFromConnectivityTests = make(map[configsections.ContainerIdentifier]interface{}) @@ -267,7 +269,7 @@ func (env *TestEnvironment) doAutodiscover() { for _, cid := range env.Config.Partner.ContainersDebugList { env.ContainersToExcludeFromConnectivityTests[cid.ContainerIdentifier] = "" } - autodiscover.FindTestPartner(&env.Config.Partner, env.NameSpaceUnderTest) + autodiscover.FindTestPartner(&env.Config.Partner, defaultNamespace) env.PartnerContainers = env.createContainers(env.Config.Partner.ContainerConfigList) env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest diff --git a/pkg/tnf/handlers/deployments/deployments.go b/pkg/tnf/handlers/deployments/deployments.go index 669cfe657..74cd4c72a 100644 --- a/pkg/tnf/handlers/deployments/deployments.go +++ b/pkg/tnf/handlers/deployments/deployments.go @@ -45,6 +45,7 @@ type DeploymentMap map[string]Deployment // Deployments holds information derived from running "oc -n get deployments" on the command line. type Deployments struct { deployments DeploymentMap + namespace string result int timeout time.Duration args []string @@ -53,8 +54,9 @@ type Deployments struct { // NewDeployments creates a new Deployments tnf.Test. func NewDeployments(timeout time.Duration, namespace string) *Deployments { return &Deployments{ - timeout: timeout, - result: tnf.ERROR, + timeout: timeout, + namespace: namespace, + result: tnf.ERROR, args: []string{"oc", "-n", namespace, "get", "deployments", "-o", "custom-columns=" + "NAME:.metadata.name," + "REPLICAS:.spec.replicas," + @@ -114,7 +116,10 @@ func (dp *Deployments) ReelMatch(_, _, match string) *reel.Step { if len(fields) != numExepctedFields { return nil } - dp.deployments[fields[0]] = Deployment{atoi(fields[1]), atoi(fields[2]), atoi(fields[3]), atoi(fields[4]), atoi(fields[5])} + // we can have the same deployment in different namespaces + // this ensures the uniqueness of the deployment in the test + key := dp.namespace + ":" + fields[0] + dp.deployments[key] = Deployment{atoi(fields[1]), atoi(fields[2]), atoi(fields[3]), atoi(fields[4]), atoi(fields[5])} } dp.result = tnf.SUCCESS diff --git a/pkg/tnf/handlers/deployments/deployments_test.go b/pkg/tnf/handlers/deployments/deployments_test.go index 5515f8134..78a6d2def 100644 --- a/pkg/tnf/handlers/deployments/deployments_test.go +++ b/pkg/tnf/handlers/deployments/deployments_test.go @@ -62,10 +62,10 @@ func Test_ReelMatchSuccess(t *testing.T) { assert.Len(t, newDp.GetDeployments(), testInputSuccessNumLines) expectedDeployments := dp.DeploymentMap{ - "cdi-apiserver": {1, 1, 1, 1, 0}, - "hyperconverged-cluster-operator": {1, 0, 1, 0, 1}, - "virt-api": {2, 2, 2, 2, 0}, - "vm-import-operator": {0, 0, 0, 0, 0}, + "testNamespace:cdi-apiserver": {1, 1, 1, 1, 0}, + "testNamespace:hyperconverged-cluster-operator": {1, 0, 1, 0, 1}, + "testNamespace:virt-api": {2, 2, 2, 2, 0}, + "testNamespace:vm-import-operator": {0, 0, 0, 0, 0}, } deployments := newDp.GetDeployments() diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 28c11ae91..4579e1318 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -279,20 +279,26 @@ func shutdownTest(podNamespace, podName string) { } func testPodsRecreation(env *config.TestEnvironment) { - var deployments dp.DeploymentMap + deployments := make(dp.DeploymentMap) var notReadyDeployments []string testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestPodRecreationIdentifier) ginkgo.It(testID, func() { ginkgo.By("Testing node draining effect of deployment") - ginkgo.By(fmt.Sprintf("test deployment in namespace %s", env.NameSpaceUnderTest)) - deployments, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) - if len(deployments) == 0 { - return + ginkgo.By(fmt.Sprintf("test deployment in namespace %s", env.NameSpacesUnderTest)) + for _, ns := range env.NameSpacesUnderTest { + var dps dp.DeploymentMap + dps, notReadyDeployments = getDeployments(ns) + for dpKey, dp := range dps { + deployments[dpKey] = dp + } + // We require that all deployments have the desired number of replicas and are all up to date + if len(notReadyDeployments) != 0 { + ginkgo.Skip("Can not test when deployments are not ready") + } } - // We require that all deployments have the desired number of replicas and are all up to date - if len(notReadyDeployments) != 0 { - ginkgo.Skip("Can not test when deployments are not ready") + if len(deployments) == 0 { + ginkgo.Skip("no valid deployment") } defer env.SetNeedsRefresh() ginkgo.By("should create new replicas when node is drained") @@ -304,23 +310,23 @@ func testPodsRecreation(env *config.TestEnvironment) { // We need to delete all Oc sessions because the drain operation is often deleting oauth-openshift pod // This result in lost connectivity oc sessions env.ResetOc() - // drain node drainNode(n.Name) // should go in this - - waitForAllDeploymentsReady(env.NameSpaceUnderTest, scalingTimeout, scalingPollingPeriod) - - // verify deployments are ready again - _, notReadyDeployments = getDeployments(env.NameSpaceUnderTest) - if len(notReadyDeployments) != 0 { - uncordonNode(n.Name) - ginkgo.Fail(fmt.Sprintf("did not create replicas when node %s is drained", n.Name)) + for _, ns := range env.NameSpacesUnderTest { + waitForAllDeploymentsReady(ns, scalingTimeout, scalingPollingPeriod) + // verify deployments are ready again + _, notReadyDeployments = getDeployments(ns) + if len(notReadyDeployments) > 0 { + uncordonNode(n.Name) + ginkgo.Fail(fmt.Sprintf("did not create replicas when node %s is drained", n.Name)) + } } - uncordonNode(n.Name) - // wait for all deployment to be ready otherwise, pods might be unreacheable during the next discovery - waitForAllDeploymentsReady(env.NameSpaceUnderTest, scalingTimeout, scalingPollingPeriod) + for _, ns := range env.NameSpacesUnderTest { + // wait for all deployment to be ready otherwise, pods might be unreacheable during the next discovery + waitForAllDeploymentsReady(ns, scalingTimeout, scalingPollingPeriod) + } } }) } diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index 3f34f42f8..bd1ab63be 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -151,10 +151,12 @@ func testNodePort(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestServicesDoNotUseNodeportsIdentifier) ginkgo.It(testID, func() { context := common.GetContext() - ginkgo.By(fmt.Sprintf("Testing services in namespace %s", env.NameSpaceUnderTest)) - tester := nodeport.NewNodePort(common.DefaultTimeout, env.NameSpaceUnderTest) - test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) - gomega.Expect(err).To(gomega.BeNil()) - test.RunAndValidate() + for _, ns := range env.NameSpacesUnderTest { + ginkgo.By(fmt.Sprintf("Testing services in namespace %s", ns)) + tester := nodeport.NewNodePort(common.DefaultTimeout, ns) + test, err := tnf.NewTest(context.GetExpecter(), tester, []reel.Handler{tester}, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() + } }) } From 42b9e627d65772b97f06c7bc87077ade47f192fa Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Thu, 11 Nov 2021 05:29:15 -0500 Subject: [PATCH 150/344] Up-version oc client in tnf image (#463) * Up-version oc client in tnf image the latest "oc adm drain" command used by the pod recreation test doesn't seem to work with oc client 4.6 * Update README.md --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b39c6783f..eb72f82f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG TNF_PARTNER_DIR=/usr/tnf-partner ENV TNF_PARTNER_SRC_DIR=$TNF_PARTNER_DIR/src -ENV OPENSHIFT_VERSION=4.6.32 +ENV OPENSHIFT_VERSION=4.7.7 ENV TNF_DIR=/usr/tnf ENV TNF_SRC_DIR=${TNF_DIR}/tnf-src diff --git a/README.md b/README.md index 5479ea2d9..23949a5d4 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ Dependency|Minimum Version [GoLang](https://golang.org/dl/)|1.17 [golangci-lint](https://golangci-lint.run/usage/install/)|1.42.1 [jq](https://stedolan.github.io/jq/)|1.6 -[OpenShift Client](https://docs.openshift.com/container-platform/4.4/welcome/index.html)|4.4 +[OpenShift Client](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/)|4.7 Other binary dependencies required to run tests can be installed using the following command: From b68b19975b3a6105ea4c2f2552e694ed37a1b121 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:11:32 +0100 Subject: [PATCH 151/344] Restore CRDs autodiscovery. (#464) The CRD autodiscovery was removed by mistake in this commit: 03b5d5c464d40279ab8bcb09ad20ab72014a056f (PR: https://github.com/test-network-function/test-network-function/pull/395) --- pkg/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4f94a703c..fd377db7c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -274,6 +274,7 @@ func (env *TestEnvironment) doAutodiscover() { env.TestOrchestrator = env.PartnerContainers[env.Config.Partner.TestOrchestratorID] env.DeploymentsUnderTest = env.Config.DeploymentsUnderTest env.OperatorsUnderTest = env.Config.Operators + env.CrdNames = autodiscover.FindTestCrdNames(env.Config.CrdFilters) env.discoverNodes() log.Infof("Test Configuration: %+v", *env) From 338e49d3167a6ae4d71dfdf4f7587083fb9f1b9d Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 12 Nov 2021 17:43:05 +0100 Subject: [PATCH 152/344] fix ifconfig (#469) --- test-network-function/diagnostic/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 5724880da..1ba7a244a 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -266,7 +266,7 @@ func getNodeLscpu(nodeName string) map[string]string { } func getNodeIfconfig(nodeName string) map[string][]string { - const command = "ifconfig" + const command = "chroot /host ifconfig" const numSplitSubstrings = 2 result := map[string][]string{} env = config.GetTestEnvironment() From dff3dd288ccfc960f2da11dc33b9982b294b9021 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 12 Nov 2021 12:18:53 -0500 Subject: [PATCH 153/344] Fix catalog md header (#471) --- CATALOG.md | 6 ++++-- cmd/tnf/generate/catalog/INTRO.md | 2 +- cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 0b4988148..19559f57a 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -2,9 +2,11 @@ test-network-function contains a variety of `Test Cases`, as well as `Test Case Building Blocks`. * Test Cases: Traditional JUnit testcases, which are specified internally using `Ginkgo.It`. Test cases often utilize several Test Case Building Blocks. -* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`.## Test Case Building Blocks Catalog +* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. +## Test Case Catalog -A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations:### http://test-network-function.com/testcases/access-control/cluster-role-bindings +Test Cases are the specifications used to perform a meaningful test. Test cases may run once, or several times against several targets. CNF Certification includes a number of normative and informative tests to ensure CNFs follow best practices. Here is the list of available Test Cases: +### http://test-network-function.com/testcases/access-control/cluster-role-bindings Property|Description ---|--- diff --git a/cmd/tnf/generate/catalog/INTRO.md b/cmd/tnf/generate/catalog/INTRO.md index 45dafde4f..7ca07b3a1 100644 --- a/cmd/tnf/generate/catalog/INTRO.md +++ b/cmd/tnf/generate/catalog/INTRO.md @@ -2,4 +2,4 @@ test-network-function contains a variety of `Test Cases`, as well as `Test Case Building Blocks`. * Test Cases: Traditional JUnit testcases, which are specified internally using `Ginkgo.It`. Test cases often utilize several Test Case Building Blocks. -* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. \ No newline at end of file +* Test Case Building Blocks: Self-contained building blocks, which perform a small task in the context of `oc`, `ssh`, `shell`, or some other `Expecter`. diff --git a/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md b/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md index 826f83544..9d87bd94a 100644 --- a/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md +++ b/cmd/tnf/generate/catalog/TEST_CASE_CATALOG.md @@ -1,3 +1,3 @@ -## Test Case Building Blocks Catalog +## Test Case Catalog -A number of Test Case Building Blocks, or `tnf.Test`s, are included out of the box. This is a summary of the available implementations: \ No newline at end of file +Test Cases are the specifications used to perform a meaningful test. Test cases may run once, or several times against several targets. CNF Certification includes a number of normative and informative tests to ensure CNFs follow best practices. Here is the list of available Test Cases: From f63211102b9566c60792f6e21f9820d2ae672443 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 12 Nov 2021 15:19:42 -0600 Subject: [PATCH 154/344] Fails Ginko test with non zero exit code (#470) * Fails Ginko test with non zero exit code * PR review Salah * PR review Jun and improving claim file with ERR * fix ifconfig (#469) * Fix catalog md header (#471) * PR Review Jun * PR Review Jun fix Co-authored-by: Salaheddine Hamadi Co-authored-by: Jun Chen --- pkg/tnf/reel/reel.go | 8 ++++++-- pkg/tnf/test.go | 8 +------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/tnf/reel/reel.go b/pkg/tnf/reel/reel.go index 94cacbbd1..8ce247639 100644 --- a/pkg/tnf/reel/reel.go +++ b/pkg/tnf/reel/reel.go @@ -197,7 +197,6 @@ func (r *Reel) Step(step *Step, handler Handler) error { if !step.hasExpectations() { return nil } - if err != nil { if isTimeout(err) { step = handler.ReelTimeout() @@ -210,10 +209,11 @@ func (r *Reel) Step(step *Step, handler Handler) error { output, outputStatus := r.stripEmulatedPromptFromOutput(result.Output) if outputStatus != 0 { - r.Err = fmt.Errorf("error executing command %d: ", outputStatus) + return fmt.Errorf("error executing command exit code:%d", outputStatus) } match, matchStatus := r.stripEmulatedPromptFromOutput(result.Match[0]) log.Debugf("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d", output, match, outputStatus, matchStatus) + matchIndex := strings.Index(output, match) var before string // special case: the match regex may be nothing at all. @@ -226,6 +226,10 @@ func (r *Reel) Step(step *Step, handler Handler) error { step = handler.ReelMatch(strippedFirstMatchRe, before, match) } } + // This is for the last step + if r.Err != nil { + return r.Err + } } return nil } diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index 1f424f7bf..8ae9126e1 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -19,8 +19,6 @@ package tnf import ( "time" - log "github.com/sirupsen/logrus" - expect "github.com/google/goexpect" "github.com/onsi/gomega" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" @@ -72,10 +70,6 @@ type Test struct { // Run performs a test, returning the result and any encountered errors. func (t *Test) Run() (int, error) { err := t.runner.Run(t) - // if the runner fails, print the error - if t.runner.Err != nil { - log.Errorf("%s", t.runner.Err) - } return t.tester.Result(), err } @@ -131,8 +125,8 @@ func (t *Test) RunAndValidateWithFailureCallback(cb func()) { if testResult == FAILURE && cb != nil { cb() } + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) gomega.Expect(testResult).To(gomega.Equal(SUCCESS)) - gomega.Expect(err).To(gomega.BeNil()) } // RunWithCallbacks runs the test, invokes the cb on failure/error/success From bced9a4ce2206e14c2bd4714816e059b90663ddf Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 12 Nov 2021 16:20:26 -0500 Subject: [PATCH 155/344] Update version.json --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 201ddc59e..4d8b11a17 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "partner_tag": "v3.0.0" + "partner_tag": "v3.1.0" } From 28793559bf40a9b7b72d338966757e1b20745a24 Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Mon, 15 Nov 2021 05:48:34 +0100 Subject: [PATCH 156/344] update readme (#473) * update readme * Update README.md * Update README.md * Update README.md Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> * fix Jun's comments * more fixes Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> Co-authored-by: Jun Chen --- README.md | 45 ++++++++++------------------ test-network-function/tnf_config.yml | 31 +------------------ 2 files changed, 16 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 23949a5d4..bb330437c 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,14 @@ The Test Network Function support autodiscovery using labels and annotations. Th ### targetNameSpaces -A single namespace should be specified in the [configuration file](test-network-function/tnf_config.yml). This namespace will be used by autodiscovery to find the Pods under test. To run multiple tests in different namespaces simultaneously, intrusive tests should be disabled by setting ``TNF_NON_INTRUSIVE_ONLY`` to true. +Multiple namespaces can be specified in the [configuration file](test-network-function/tnf_config.yml). Namespaces will be used by autodiscovery to find the Pods under test. +``` shell script +targetNameSpaces: + - name: firstnamespace + - name: secondnamespace +``` ### targetPodLabels -The goal of this section is to specify the label to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The prefix field can be used to avoid naming collision with other labels. +The goal of this section is to specify the labels to be used to identify the CNF resources under test. It's highly recommended that the labels should be defined in pod definition rather than added after pod is created, as labels added later on will be lost in case the pod gets rescheduled. In case of pods defined as part of a deployment, it's best to use the same label as the one defined in the `spec.selector.matchLabels` section of the deployment yaml. The prefix field can be used to avoid naming collision with other labels. ```shell script targetPodLabels: - prefix: test-network-function.com @@ -56,32 +61,19 @@ The autodiscovery mechanism will create a list of all CRD names in the cluster w ### testTarget #### podsUnderTest / containersUnderTest -This section is usually not required if labels defined in the section above cover all resources that should be tested. If label based discovery is not sufficient, this section can be manually populated as shown in the commented part of the [sample config](test-network-function/tnf_config.yml). However, intrusive tests need to be skipped ([see here](#disable-intrusive-tests)) for a reliable test result. The pods and containers explicitly configured here are added to the target pod/container lists populated through label matching. - -For both configured and discovered pods/containers, the autodiscovery mechanism will attempt to identify the default network device and all the IP addresses of the pods it -needs for network connectivity tests, though that information can be explicitly set using annotations if needed. For Pod IPs: +The autodiscovery mechanism will attempt to identify the default network device and all the IP addresses of the pods it needs for network connectivity tests, though that information can be explicitly set using annotations if needed. For Pod IPs: -* The annotation `test-network-function.com/multusips` is the highest priority, and must contain a JSON-encoded list of -IP addresses to be tested for the pod. This must be explicitly set. -* If the above is not present, the `k8s.v1.cni.cncf.io/networks-status` annotation is checked and all IPs from it are -used. This annotation is automatically managed in OpenShift but may not be present in K8s. -* If neither of the above is present, then only known IPs associated with the pod are used (the pod `.status.ips` field). +* The annotation test-network-function.com/multusips is the highest priority, and must contain a JSON-encoded list of IP addresses to be tested for the pod. This must be explicitly set. +* If the above is not present, the k8s.v1.cni.cncf.io/networks-status annotation is checked and all IPs from it are used. This annotation is automatically managed in OpenShift but may not be present in K8s. +* If neither of the above is present, then only known IPs associated with the pod are used (the pod .status.ips field). For Network Interfaces: -* The annotation `test-network-function.com/defaultnetworkinterface` is the highest priority, and must contain a -JSON-encoded string of the primary network interface for the pod. This must be explicitly set if needed. Examples can -be seen in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/local-test-infra/local-pod-under-test/local-partner-pod.yaml) -* If the above is not present, the `k8s.v1.cni.cncf.io/networks-status` annotation is checked and the `"interface"` from -the first entry found with `"default"=true` is used. This annotation is automatically managed in OpenShift but may not -be present in K8s. - -If multus IP addresses are discovered or configured, the partner pod needs to be deployed in the same namespace as the multus network interface for the connectivity test to pass. Refer to instruction [here](#specify-the-target-namespace-for-partner-pod-deployment). - -If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. `ping`), it should be -given the label `test-network-function.com/skip_connectivity_tests` to exclude it from those tests. The label value is -not important, only its presence. Equivalent to `excludeContainersFromConnectivityTests` in the config file. +* The annotation test-network-function.com/defaultnetworkinterface is the highest priority, and must contain a JSON-encoded string of the primary network interface for the pod. This must be explicitly set if needed. Examples can be seen in cnf-certification-test-partner +* If the above is not present, the k8s.v1.cni.cncf.io/networks-status annotation is checked and the "interface" from the first entry found with "default"=true is used. This annotation is automatically managed in OpenShift but may not be present in K8s. +* If multus IP addresses are discovered or configured, the partner pod needs to be deployed in the same namespace as the multus network interface for the connectivity test to pass. Refer to instruction here. +If a pod is not suitable for network connectivity tests because it lacks binaries (e.g. ping), it should be given the label test-network-function.com/skip_connectivity_tests to exclude it from those tests. The label value is not important, only its presence. #### operators @@ -133,13 +125,6 @@ export TNF_PARTNER_SRC_DIR=/home/userid/code/cnf-certification-test-partner When this variable is set, the run-cnf-suites.sh script will deploy/refresh the partner deployments/pods in the cluster before starting the test run. -### Specify the target namespace for partner pod deployment -Set this variable to deploy partner pods in a custom namespace instead of the default `tnf` namespace. - -```shell-script -export TNF_PARTNER_NAMESPACE="CNF-ns" -``` - ### Disconnected environment In disconnected environment, only specific versions of images are mirrored to the local repo. For the `oc debug` command (used by a number of tests) to work, set TNF_OC_DEBUG_IMAGE_ID: diff --git a/test-network-function/tnf_config.yml b/test-network-function/tnf_config.yml index e8f0e139d..8afd0f719 100644 --- a/test-network-function/tnf_config.yml +++ b/test-network-function/tnf_config.yml @@ -7,31 +7,6 @@ targetPodLabels: targetCrdFilters: - nameSuffix: "group1.test.com" - nameSuffix: "test-network-function.com" -# The following section does not require manual configuration as autodiscovery is on by default -# Containers and pods will be found through matching targetPodLabels. Operators will be found if -# labelled with "test-network-function.com/operator". Their subscription name will be read from -# annotation named "subscription_name". -# -# To add additional test targets, uncomment the section and configuree the values -# -# testTarget: -# containersUnderTest: -# - namespace: tnf -# podName: test -# containerName: test -# defaultNetworkDevice: eth0 -# multusIpAddresses: -# - 10.217.0.8 -# podsUnderTest: -# - name: test -# namespace: tnf -# operators: -# - name: etcdoperator.v0.9.4 -# namespace: default -# subscriptionName: etcd -# autogenerate: false - - # The following section does not require manual configuration as autodiscovery is on by default. # Partner pods deployed automatically from the cnf-test-partner-repo should have all the labels # used by autodiscovery @@ -47,11 +22,7 @@ targetCrdFilters: # - namespace: tnf # podName: node-master # containerName: master -# defaultNetworkDevice: eth0 -# fsDiffMasterContainer: -# namespace: tnf -# podName: node-master -# containerName: master +# defaultNetworkDevice: eth0 # testOrchestrator: # namespace: tnf # podName: partner From fa89defe04b1ee1f654a1af8f5b1c52ec3f625ee Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Mon, 15 Nov 2021 11:35:20 -0600 Subject: [PATCH 157/344] Use go template for hostPort check (#474) * Use go template for hostPort check * Address comments 1 --- pkg/tnf/testcases/data/cnf/privilegedpod.go | 3 ++- pkg/tnf/testcases/files/cnf/privilegedpod.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/tnf/testcases/data/cnf/privilegedpod.go b/pkg/tnf/testcases/data/cnf/privilegedpod.go index 4942a4830..57baf5e6d 100644 --- a/pkg/tnf/testcases/data/cnf/privilegedpod.go +++ b/pkg/tnf/testcases/data/cnf/privilegedpod.go @@ -17,6 +17,7 @@ package cnf // PrivilegedPodJSON test templates for privileged pods +//nolint:lll var PrivilegedPodJSON = string(`{ "testcase": [ { @@ -33,7 +34,7 @@ var PrivilegedPodJSON = string(`{ "name": "HOST_PORT_CHECK", "skiptest": true, "loop": 1, - "command": "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d] | select(.ports) | .ports[].hostPort'", + "command": "oc get pod %s -n %s -o go-template='{{$putName := .metadata.name}}{{$cut := (index .spec.containers %d)}}{{range $cut.ports }}{{if .hostPort}}PUT {{$putName}} - CUT {{$cut.name}} has declared hostPort {{.hostPort}}{{\"\\n\"}}{{end}}{{end}}'", "action": "allow", "expectedstatus": [ "^()*$" diff --git a/pkg/tnf/testcases/files/cnf/privilegedpod.yml b/pkg/tnf/testcases/files/cnf/privilegedpod.yml index dbe154706..0515b0b3d 100644 --- a/pkg/tnf/testcases/files/cnf/privilegedpod.yml +++ b/pkg/tnf/testcases/files/cnf/privilegedpod.yml @@ -10,7 +10,7 @@ testcase: - name: HOST_PORT_CHECK skiptest: true loop: 0 - command: "oc get pod %s -n %s -o json | jq -r '.spec.containers[%d] | select(.ports) | .ports[].hostPort'" + command: "oc get pod %s -n %s -o go-template='{{$putName := .metadata.name}}{{$cut := (index .spec.containers %d)}}{{range $cut.ports }}{{if .hostPort}}PUT {{$putName}} - CUT {{$cut.name}} has declared hostPort {{.hostPort}}{{\"\\n\"}}{{end}}{{end}}'" action: allow expectedType: "regex" expectedstatus: From c8521d244b77ea5bf163ee67812729000b2fade1 Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Mon, 15 Nov 2021 16:15:29 -0600 Subject: [PATCH 158/344] remove 3.0.x branch(due to failure to build), add 3.1.x branch to matrix (#477) --- .github/workflows/tnf-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index 7b5da5d65..e9993bf4f 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - branch: [3.0.x] + branch: [3.1.x] env: SHELL: /bin/bash KUBECONFIG: '/home/runner/.kube/config' From 42d03b679515258aae80b024ebcaa18afda5683d Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 16 Nov 2021 11:48:42 -0600 Subject: [PATCH 159/344] GolangCI v1.43.0; various fixes (#476) --- Makefile | 4 ++-- README.md | 2 +- internal/api/api.go | 6 +++--- internal/api/api_test.go | 4 ++-- pkg/config/autodiscover/deployment_info.go | 2 +- pkg/config/configsections/certified_request_test.go | 3 +-- pkg/config/configsections/config_section_test.go | 5 ++--- pkg/gradetool/gradetool.go | 8 ++++---- pkg/tnf/testcases/base.go | 2 +- pkg/tnf/testcases/base_test.go | 3 +-- 10 files changed, 18 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 08cc7dc14..8b3d03a45 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ COMMON_GO_ARGS=-race GIT_COMMIT=$(shell script/create-version-files.sh) GIT_RELEASE=$(shell script/get-git-release.sh) GIT_PREVIOUS_RELEASE=$(shell script/get-git-previous-release.sh) -GOLANGCI_VERSION=v1.42.1 +GOLANGCI_VERSION=v1.43.0 # Run the unit tests and build all binaries build: @@ -72,7 +72,7 @@ clean: # Run configured linters lint: - golangci-lint run + golangci-lint run --timeout 5m0s # Build and run unit tests test: mocks diff --git a/README.md b/README.md index bb330437c..9b9696d9c 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ At a minimum, the following dependencies must be installed *prior* to running `m Dependency|Minimum Version ---|--- [GoLang](https://golang.org/dl/)|1.17 -[golangci-lint](https://golangci-lint.run/usage/install/)|1.42.1 +[golangci-lint](https://golangci-lint.run/usage/install/)|1.43.0 [jq](https://stedolan.github.io/jq/)|1.6 [OpenShift Client](https://mirror.openshift.com/pub/openshift-v4/clients/ocp/)|4.7 diff --git a/internal/api/api.go b/internal/api/api.go index 0d1a594fc..5a83f9b75 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -3,7 +3,7 @@ package api import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" log "github.com/sirupsen/logrus" @@ -94,7 +94,7 @@ func (api CertAPIClient) GetOperatorBundleIDByPackageName(org, name string) (ima // getRequest a http call to rest api, returns byte array or error func (api CertAPIClient) getRequest(url string) (response []byte, err error) { - req, err := http.NewRequest(http.MethodGet, url, nil) //nolint:noctx + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) //nolint:noctx if err != nil { return nil, err } @@ -107,7 +107,7 @@ func (api CertAPIClient) getRequest(url string) (response []byte, err error) { err = GetContainer404Error() return } - if response, err = ioutil.ReadAll(resp.Body); err != nil { + if response, err = io.ReadAll(resp.Body); err != nil { err = GetContainer404Error() return } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 95f60b91e..737bea011 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -3,7 +3,7 @@ package api_test import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "testing" @@ -102,7 +102,7 @@ func (m *MockClient) Do(req *http.Request) (*http.Response, error) { } func getDoFunc(data string, status int) func(req *http.Request) (*http.Response, error) { - response := ioutil.NopCloser(bytes.NewReader([]byte(data))) + response := io.NopCloser(bytes.NewReader([]byte(data))) defer response.Close() return func(*http.Request) (*http.Response, error) { return &http.Response{ diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index 6c09a3810..c8f769dd1 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -80,7 +80,7 @@ func (deployment *DeploymentResource) GetLabels() map[string]string { // GetTargetDeploymentsByNamespace will return all deployments that have pods with a given label. func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsections.Label) (*DeploymentList, error) { - labelQuery := fmt.Sprintf("\"%s\"==\"%s\"", buildLabelName(targetLabel.Prefix, targetLabel.Name), targetLabel.Value) + labelQuery := fmt.Sprintf("%q==%q", buildLabelName(targetLabel.Prefix, targetLabel.Name), targetLabel.Value) jqArgs := fmt.Sprintf("'[.items[] | select(.spec.template.metadata.labels.%s)]'", labelQuery) ocCmd := fmt.Sprintf("oc get %s -n %s -o json | jq %s", resourceTypeDeployment, namespace, jqArgs) diff --git a/pkg/config/configsections/certified_request_test.go b/pkg/config/configsections/certified_request_test.go index 0f44416d8..27f4df988 100644 --- a/pkg/config/configsections/certified_request_test.go +++ b/pkg/config/configsections/certified_request_test.go @@ -18,7 +18,6 @@ package configsections import ( "encoding/json" - "io/ioutil" "log" "os" "testing" @@ -66,7 +65,7 @@ var ( // setupRequestTest writes the result of `populateRequestConfig` to a temporary file for loading in a test. func setupRequestTest(marshalFun marshalFunc) (tempfileName string) { - tempfile, err := ioutil.TempFile(".", tmpfileNameBase) + tempfile, err := os.CreateTemp(".", tmpfileNameBase) if err != nil { log.Fatal(err) } diff --git a/pkg/config/configsections/config_section_test.go b/pkg/config/configsections/config_section_test.go index b2805706f..68ae78a79 100644 --- a/pkg/config/configsections/config_section_test.go +++ b/pkg/config/configsections/config_section_test.go @@ -18,7 +18,6 @@ package configsections import ( "encoding/json" - "io/ioutil" "log" "os" "testing" @@ -145,7 +144,7 @@ func loadFullConfig() { } func setup(configType string) { - file, err = ioutil.TempFile(".", "test-config.yml") + file, err = os.CreateTemp(".", "test-config.yml") if err != nil { log.Fatal(err) } @@ -166,7 +165,7 @@ func setup(configType string) { } func setupJSON(configType string) { - jsonFile, err = ioutil.TempFile(".", "test-json-config.json") + jsonFile, err = os.CreateTemp(".", "test-json-config.json") if err != nil { log.Fatal(err) } diff --git a/pkg/gradetool/gradetool.go b/pkg/gradetool/gradetool.go index e3f98a272..286e5302b 100644 --- a/pkg/gradetool/gradetool.go +++ b/pkg/gradetool/gradetool.go @@ -19,7 +19,7 @@ package gradetool import ( "encoding/json" "fmt" - "io/ioutil" + "os" "path" "github.com/test-network-function/test-network-function-claim/pkg/claim" @@ -102,7 +102,7 @@ func NewGradeResult(gradeName string) GradeResult { } func generateTestResultsKey(id identifier.Identifier) string { - return fmt.Sprintf("{\"url\":\"%s\",\"version\":\"%s\"}", id.URL, id.SemanticVersion) + return fmt.Sprintf("{\"url\":%q,\"version\":%q}", id.URL, id.SemanticVersion) } func doGrading(policy Policy, results map[string]interface{}) (interface{}, error) { @@ -171,7 +171,7 @@ func generateOutput(outputObj interface{}, outputPath string) error { if err != nil { return err } - err = ioutil.WriteFile(outputPath, outputBytes, outputFilePermissions) + err = os.WriteFile(outputPath, outputBytes, outputFilePermissions) if err != nil { return err } @@ -206,7 +206,7 @@ func validatePolicySchema(policyPath string) error { } func unmarshalFromFile(jsonPath string, obj interface{}) error { - jsonBytes, err := ioutil.ReadFile(jsonPath) + jsonBytes, err := os.ReadFile(jsonPath) if err != nil { return err } diff --git a/pkg/tnf/testcases/base.go b/pkg/tnf/testcases/base.go index 77f5df053..7e328b185 100644 --- a/pkg/tnf/testcases/base.go +++ b/pkg/tnf/testcases/base.go @@ -341,7 +341,7 @@ func IsInFocus(focus []string, desc string) bool { focusFilter = regexp.MustCompile(strings.Join(focus, "|")) } if focusFilter != nil { - matchesFocus = focusFilter.Match([]byte(desc)) + matchesFocus = focusFilter.MatchString(desc) } return matchesFocus } diff --git a/pkg/tnf/testcases/base_test.go b/pkg/tnf/testcases/base_test.go index b061f5059..21edc274d 100644 --- a/pkg/tnf/testcases/base_test.go +++ b/pkg/tnf/testcases/base_test.go @@ -17,7 +17,6 @@ package testcases_test import ( - "io/ioutil" "log" "os" "reflect" @@ -53,7 +52,7 @@ func setup() { configuredTest.Tests = []string{"HOST_NETWORK_CHECK", "HOST_PORT_CHECK", "HOST_IPC_CHECK"} testConfigure.CnfTest = append(testConfigure.CnfTest, configuredTest) - file, err = ioutil.TempFile(".", testTempFile) + file, err = os.CreateTemp(".", testTempFile) if err != nil { log.Fatal(err) } From a33b3d5d5960524c964dd1f9be0211996e9e0615 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Wed, 17 Nov 2021 07:58:55 +0100 Subject: [PATCH 160/344] Escape special chars in the utils.ExecuteCommand function. (#467) Apart from the double quote, the new line char also needs to be escaped so the command doesn't break the json string value for the generic Command handler. Example: "oc get ... -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' The {{"\n"}} needs to be replaced to {{\"\\n\"}} Added function to escape string to json format. Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- pkg/utils/utils.go | 21 +++++++++++++++++++-- pkg/utils/utils_test.go | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 pkg/utils/utils_test.go diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 81298947e..765ab65c4 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/json" "errors" "os" "path" @@ -61,16 +62,32 @@ func CheckFileExists(filePath, name string) { } } +func escapeToJSONstringFormat(line string) (string, error) { + // Newlines need manual escaping. + line = strings.ReplaceAll(line, "\n", "\\n") + marshalled, err := json.Marshal(line) + if err != nil { + return "", err + } + s := string(marshalled) + // Remove double quotes and return marshalled string. + return s[1 : len(s)-1], nil +} + // ExecuteCommand uses the generic command handler to execute an arbitrary interactive command, returning // its output wihout any other check. func ExecuteCommand(command string, timeout time.Duration, context *interactive.Context, failureCallbackFun func()) string { log.Debugf("Executing command: %s", command) values := make(map[string]interface{}) - // Escapes the double quote char to make a valid json string. - values["COMMAND"] = strings.ReplaceAll(command, "\"", "\\\"") + // Escapes the double quote and new line chars to make a valid json string for the command to be executed by the handler. + var err error + values["COMMAND"], err = escapeToJSONstringFormat(command) + gomega.Expect(err).To(gomega.BeNil()) values["TIMEOUT"] = timeout.Nanoseconds() + log.Debugf("Command handler's COMMAND string value: %s", values["COMMAND"]) + tester, handler, result, err := generic.NewGenericFromMap(commandHandlerFilePath, handlerJSONSchemaFilePath, values) gomega.Expect(err).To(gomega.BeNil()) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 000000000..e49bbef1c --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,18 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + testString1 = "{{\"Quoted line with new line\n char and also some others \b chars not commonly used like \f, \t, \\ and \r.\"}}" + testEscapedString1 = `{{\"Quoted line with new line\\n char and also some others \u0008 chars not commonly used like \u000c, \t, \\ and \r.\"}}` +) + +func TestEscapeToJSONstringFormat(t *testing.T) { + escapedString, err := escapeToJSONstringFormat(testString1) + assert.Nil(t, err) + assert.Equal(t, testEscapedString1, escapedString) +} From c46bf49e9c9be70c6f7b494b1aab8a16008c4327 Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Wed, 17 Nov 2021 19:37:53 +0200 Subject: [PATCH 161/344] CNFCERT-170:HPA support (autodiscovery and scaling) (#459) * CNFCERT-170:HPA support (autodiscovery and scaling) * CNFCERT-170:HPA support (autodiscovery and scaling) * CNFCERT-170:HPA fix hpa struct * CNFCERT-170:HPA fix hpa struct * CNFCERT-170:HPA fix hpa struct * CNFCERT-170:HPA fix hpa struct * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:HPA few fixes * CNFCERT-170:adding new handler for the hpa and adding a test for it * CNFCERT-170:udpating the hpascaling function name * CNFCERT-170:udpating hte hpa name * upadate the q * upadate the restore deployment * upadate the restore deployment * upadate the restore deployment * upadate the restore deployment Co-authored-by: Jun Chen --- .../autodiscover/autodiscover_targets.go | 1 + pkg/config/autodiscover/deployment_info.go | 19 ++++ pkg/config/configsections/deployment.go | 1 + pkg/config/configsections/hpa.go | 7 ++ pkg/tnf/handlers/scaling/scaling_hpa.go | 95 +++++++++++++++++++ pkg/tnf/handlers/scaling/scaling_hpa_test.go | 79 +++++++++++++++ test-network-function/lifecycle/suite.go | 38 ++++++-- 7 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 pkg/config/configsections/hpa.go create mode 100644 pkg/tnf/handlers/scaling/scaling_hpa.go create mode 100644 pkg/tnf/handlers/scaling/scaling_hpa_test.go diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index c66d292db..8a0241b36 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -138,6 +138,7 @@ func FindTestDeployments(targetLabels []configsections.Label, target *configsect Name: deploymentResource.GetName(), Namespace: deploymentResource.GetNamespace(), Replicas: deploymentResource.GetReplicas(), + Hpa: deploymentResource.GetHpa(), } deployments = append(deployments, deployment) diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index c8f769dd1..8268702bf 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -19,6 +19,8 @@ package autodiscover import ( "encoding/json" "fmt" + "strconv" + "strings" log "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/config/configsections" @@ -77,6 +79,23 @@ func (deployment *DeploymentResource) GetReplicas() int { func (deployment *DeploymentResource) GetLabels() map[string]string { return deployment.Metadata.Labels } +func (deployment *DeploymentResource) GetHpa() configsections.Hpa { + template := fmt.Sprintf("go-template='{{ range .items }}{{ if eq .spec.scaleTargetRef.name %q }}{{.spec.minReplicas}},{{.spec.maxReplicas}},{{.metadata.name}}{{ end }}{{ end }}'", deployment.GetName()) + ocCmd := fmt.Sprintf("oc get hpa -n %s -o %s", deployment.GetNamespace(), template) + out := execCommandOutput(ocCmd) + if out != "" { + out := strings.Split(out, ",") + min, _ := strconv.Atoi(out[0]) + max, _ := strconv.Atoi(out[1]) + hpaNmae := out[2] + return configsections.Hpa{ + MinReplicas: min, + MaxReplicas: max, + HpaName: hpaNmae, + } + } + return configsections.Hpa{} +} // GetTargetDeploymentsByNamespace will return all deployments that have pods with a given label. func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsections.Label) (*DeploymentList, error) { diff --git a/pkg/config/configsections/deployment.go b/pkg/config/configsections/deployment.go index 31d698a31..0dc3381be 100644 --- a/pkg/config/configsections/deployment.go +++ b/pkg/config/configsections/deployment.go @@ -21,4 +21,5 @@ type Deployment struct { Name string Namespace string Replicas int + Hpa Hpa } diff --git a/pkg/config/configsections/hpa.go b/pkg/config/configsections/hpa.go new file mode 100644 index 000000000..61dd92ef2 --- /dev/null +++ b/pkg/config/configsections/hpa.go @@ -0,0 +1,7 @@ +package configsections + +type Hpa struct { + MinReplicas int + MaxReplicas int + HpaName string +} diff --git a/pkg/tnf/handlers/scaling/scaling_hpa.go b/pkg/tnf/handlers/scaling/scaling_hpa.go new file mode 100644 index 000000000..ed4836f2d --- /dev/null +++ b/pkg/tnf/handlers/scaling/scaling_hpa.go @@ -0,0 +1,95 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package scaling + +import ( + "fmt" + "strings" + "time" + + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" +) + +const ( + hpaOcCommand = "oc patch hpa %s -p '{\"spec\":{\"minReplicas\": %d, \"maxReplicas\": %d}}' -n %s" + hpaRegex = "horizontalpodautoscaler.autoscaling/%s patched" +) + +// Scaling holds the Scaling handler parameters. +type HpAScaling struct { + result int + timeout time.Duration + args []string + regex string +} + +// NewScaling creates a new Scaling handler. +func NewHpaScaling(timeout time.Duration, namespace, hpaName string, min, max int) *HpAScaling { + command := fmt.Sprintf(hpaOcCommand, hpaName, min, max, namespace) + return &HpAScaling{ + timeout: timeout, + result: tnf.ERROR, + args: strings.Fields(command), + regex: fmt.Sprintf(hpaRegex, hpaName), + } +} + +// Args returns the command line args for the test. +func (hpascaling *HpAScaling) Args() []string { + return hpascaling.args +} + +// GetIdentifier returns the tnf.Test specific identifiesa. +func (hpascaling *HpAScaling) GetIdentifier() identifier.Identifier { + return identifier.ScalingIdentifier +} + +// Timeout returns the timeout in seconds for the test. +func (hpascaling *HpAScaling) Timeout() time.Duration { + return hpascaling.timeout +} + +// Result returns the test result. +func (hpascaling *HpAScaling) Result() int { + return hpascaling.result +} + +// ReelFirst returns a step which expects the scale command output within the test timeout. +func (hpascaling *HpAScaling) ReelFirst() *reel.Step { + return &reel.Step{ + Execute: "", + Expect: []string{hpascaling.regex}, + Timeout: hpascaling.timeout, + } +} + +// ReelMatch does nothing, just set the test result as success. +func (hpascaling *HpAScaling) ReelMatch(_, _, match string) *reel.Step { + hpascaling.result = tnf.SUCCESS + return nil +} + +// ReelTimeout does nothing; no action is necessary upon timeout. +func (hpascaling *HpAScaling) ReelTimeout() *reel.Step { + return nil +} + +// ReelEOF does nothing; no action is necessary upon EOF. +func (hpascaling *HpAScaling) ReelEOF() { +} diff --git a/pkg/tnf/handlers/scaling/scaling_hpa_test.go b/pkg/tnf/handlers/scaling/scaling_hpa_test.go new file mode 100644 index 000000000..5429f5f5f --- /dev/null +++ b/pkg/tnf/handlers/scaling/scaling_hpa_test.go @@ -0,0 +1,79 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package scaling_test + +import ( + "fmt" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" +) + +func Test_HpaNewScaling(t *testing.T) { + handler := scaling.NewHpaScaling(testTimeoutDuration, testPodNamespace, testHpaName, testMinReplicaCount, testMaxReplicaCount) + assert.NotNil(t, handler) + assert.Equal(t, testTimeoutDuration, handler.Timeout()) + assert.Equal(t, handler.Result(), tnf.ERROR) +} + +func Test_ReelHpaFirstPositive(t *testing.T) { + handler := scaling.NewHpaScaling(testTimeoutDuration, testPodNamespace, testHpaName, testMinReplicaCount, testMaxReplicaCount) + assert.NotNil(t, handler) + firstStep := handler.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + + matches := re.FindStringSubmatch(testHpaInputSuccess) + assert.Len(t, matches, 1) +} + +func Test_ReelHpaFirstNegative(t *testing.T) { + handler := scaling.NewHpaScaling(testTimeoutDuration, testPodNamespace, testHpaName, testMinReplicaCount, testMaxReplicaCount) + assert.NotNil(t, handler) + firstStep := handler.ReelFirst() + re := regexp.MustCompile(firstStep.Expect[0]) + matches := re.FindStringSubmatch(testInputError) + assert.Len(t, matches, 0) +} + +func Test_ReelHpaMatchSuccess(t *testing.T) { + handler := scaling.NewHpaScaling(testTimeoutDuration, testPodNamespace, testHpaName, testMinReplicaCount, testMaxReplicaCount) + assert.NotNil(t, handler) + + step := handler.ReelMatch("", "", testHpaInputSuccess) + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, handler.Result()) +} + +// Just ensure there are no panics. +func Test_ReelHpaEof(t *testing.T) { + handler := scaling.NewHpaScaling(testTimeoutDuration, testPodNamespace, testHpaName, testMinReplicaCount, testMaxReplicaCount) + assert.NotNil(t, handler) + handler.ReelEOF() +} + +const ( + testMaxReplicaCount = 5 + testMinReplicaCount = 2 + testHpaName = "testHpaName" +) + +var ( + testHpaInputSuccess = fmt.Sprintf("horizontalpodautoscaler.autoscaling/%s patched\n", testHpaName) +) diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index 4579e1318..bddef1622 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -132,6 +132,9 @@ func restoreDeployments(env *config.TestEnvironment) { waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } + if deployment.Hpa.HpaName != "" { // it have hpa and need to update the max min + runHpaScalingTest(deployment) + } if deployments[deployment.Name].Replicas != deployment.Replicas { log.Warn("Deployment ", deployment.Name, " replicaCount (", deployment.Replicas, ") needs to be restored.") @@ -167,6 +170,16 @@ func runScalingTest(deployment configsections.Deployment) { waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) } +func runHpaScalingTest(deployment configsections.Deployment) { + handler := scaling.NewHpaScaling(common.DefaultTimeout, deployment.Namespace, deployment.Hpa.HpaName, deployment.Hpa.MinReplicas, deployment.Hpa.MaxReplicas) + test, err := tnf.NewTest(common.GetContext().GetExpecter(), handler, []reel.Handler{handler}, common.GetContext().GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + test.RunAndValidate() + + // Wait until the deployment is ready + waitForAllDeploymentsReady(deployment.Namespace, scalingTimeout, scalingPollingPeriod) +} + func testScaling(env *config.TestEnvironment) { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestScalingIdentifier) ginkgo.It(testID, func() { @@ -183,14 +196,23 @@ func testScaling(env *config.TestEnvironment) { closeOcSessionsByDeployment(env.ContainersUnderTest, deployment) replicaCount := deployment.Replicas - - // ScaleIn, removing one pod from the replicaCount - deployment.Replicas = replicaCount - 1 - runScalingTest(deployment) - - // Scaleout, restoring the original replicaCount number - deployment.Replicas = replicaCount - runScalingTest(deployment) + hpa := deployment.Hpa + if hpa.HpaName != "" { + deployment.Hpa.MinReplicas = replicaCount - 1 + deployment.Hpa.MaxReplicas = replicaCount - 1 + runHpaScalingTest(deployment) // scale in + deployment.Hpa.MinReplicas = replicaCount + deployment.Hpa.MaxReplicas = replicaCount + runHpaScalingTest(deployment) // scale out + } else { + // ScaleIn, removing one pod from the replicaCount + deployment.Replicas = replicaCount - 1 + runScalingTest(deployment) + + // Scaleout, restoring the original replicaCount number + deployment.Replicas = replicaCount + runScalingTest(deployment) + } } }) } From 981d8ab9678899b51430ba56815214fa77a1c817 Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Thu, 18 Nov 2021 20:02:08 +0200 Subject: [PATCH 162/344] delete hpa from pods and run scaling test (#478) * delete hpa from pods and run scaling test * fixx thre ymllint * fix ymlint * fix ymlint * fix ymlint --- .github/workflows/pre-main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index c05b10da8..bc0e085e3 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -168,9 +168,11 @@ jobs: - name: 'Test: Run generic test suite in a TNF container' run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator + + - name: 'Test: Delete HPA from pods and Run the scaling test' + run: oc delete hpa test -n tnf && ./run-cnf-suites.sh --focus lifecycle --skip lifecycle-pod-scheduling lifecycle-pod-termination-grace-period lifecycle-container-shutdown lifecycle-pod-high-availability lifecycle-pod-recreation lifecycle-pod-owner-type # Push the new unstable TNF image to Quay.io. - - name: (if on main and upstream) Authenticate against Quay.io if: ${{ github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }} uses: docker/login-action@v1 From d7e84ed20b2c3662062078d82e650620f22b0c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Pe=C3=B1a?= Date: Fri, 19 Nov 2021 10:40:37 +0100 Subject: [PATCH 163/344] Fix URL for node selector identifier (#480) It was pointing to the nodehugepagesIdentifierURL, instead of the nodeselector IdentifierURL. --- CATALOG.md | 2 +- pkg/tnf/identifier/identifiers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 19559f57a..17fe6ac77 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -560,7 +560,7 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`oc` -### http://test-network-function.com/tests/nodehugepages +### http://test-network-function.com/tests/nodeselector Property|Description ---|--- Version|v1.0.0 diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 1081ec061..8c9a42657 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -790,7 +790,7 @@ var OwnersIdentifier = Identifier{ // NodeSelectorIdentifier is the Identifier used to represent the generic NodeSelector test. var NodeSelectorIdentifier = Identifier{ - URL: nodehugepagesIdentifierURL, + URL: nodeselectorIdentifierURL, SemanticVersion: versionOne, } From 5b10b68a09b320fc8274bd7b1eb44dace6ea2603 Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Fri, 19 Nov 2021 17:43:18 +0200 Subject: [PATCH 164/344] cnfcert170: move the delete to be befor running with container] (#485) --- .github/workflows/pre-main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index bc0e085e3..3210492b0 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -166,12 +166,12 @@ jobs: - name: 'Test: Run diagnostic test suite in a TNF container' run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f diagnostic - - name: 'Test: Run generic test suite in a TNF container' - run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator - - name: 'Test: Delete HPA from pods and Run the scaling test' - run: oc delete hpa test -n tnf && ./run-cnf-suites.sh --focus lifecycle --skip lifecycle-pod-scheduling lifecycle-pod-termination-grace-period lifecycle-container-shutdown lifecycle-pod-high-availability lifecycle-pod-recreation lifecycle-pod-owner-type + run: oc delete hpa test -n tnf + - name: 'Test: Run generic test suite in a TNF container' + run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator + # Push the new unstable TNF image to Quay.io. - name: (if on main and upstream) Authenticate against Quay.io if: ${{ github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }} From 3f27833cb9d54f5833c2c7f44760fc4a3f62710c Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Fri, 19 Nov 2021 11:20:59 -0600 Subject: [PATCH 165/344] Fix Multus log (#479) Co-authored-by: Jun Chen --- test-network-function/networking/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index bd1ab63be..d39f97bd4 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -124,7 +124,7 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { testOrchestrator := env.TestOrchestrator ginkgo.By(fmt.Sprintf("a Ping is issued from %s(%s) to %s(%s) %s", testOrchestrator.Oc.GetPodName(), testOrchestrator.Oc.GetPodContainerName(), cut.Oc.GetPodName(), cut.Oc.GetPodContainerName(), - cut.DefaultNetworkIPAddress)) + multusIPAddress)) testPing(testOrchestrator.Oc, multusIPAddress, count) } if !found { From b8ff30ea2fad77a48944c1e60ec5f6925bead3af Mon Sep 17 00:00:00 2001 From: Salaheddine Hamadi Date: Fri, 19 Nov 2021 21:30:30 +0100 Subject: [PATCH 166/344] Optimize test target discovery with multiple namespaces (#482) * change some functions signature * query pods by label only * rename function * find csv by label * rename function that gets deployment * last commit to use optimized lookup for deployment * fix lint * Revert "fix lint" This reverts commit a09b56f18b3d50a113a42765a738297f3bf8ef66. * fix lint Co-authored-by: Jun Chen --- pkg/config/autodiscover/autodiscover.go | 35 ++++++-- pkg/config/autodiscover/autodiscover_debug.go | 2 +- .../autodiscover/autodiscover_partner.go | 2 +- .../autodiscover/autodiscover_targets.go | 90 +++++++++++++------ pkg/config/autodiscover/csv_info.go | 21 ++++- pkg/config/autodiscover/deployment_info.go | 17 ++++ pkg/config/autodiscover/pod_info.go | 31 ++++++- pkg/config/configsections/common.go | 3 + 8 files changed, 161 insertions(+), 40 deletions(-) diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index eb3fd398f..403571e01 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -35,6 +35,7 @@ const ( // anyLabelValue is the value that will allow any value for a label when building the label query. anyLabelValue = "" ocCommand = "oc get %s -n %s -o json -l %s" + ocAllCommand = "oc get %s -A -o json -l %s" ocCommandTimeOut = time.Second * 10 ) @@ -75,9 +76,29 @@ func executeOcGetCommand(resourceType, labelQuery, namespace string) string { return match } +func executeOcGetAllCommand(resourceType, labelQuery string) string { + ocCommandToExecute := fmt.Sprintf(ocAllCommand, resourceType, labelQuery) + match := utils.ExecuteCommand(ocCommandToExecute, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { + log.Error("can't run command: ", ocCommandToExecute) + }) + return match +} + // getContainersByLabel builds `config.Container`s from containers in pods matching a label. -func getContainersByLabel(label configsections.Label, namespace string) (containers []configsections.ContainerConfig, err error) { - pods, err := GetPodsByLabel(label, namespace) +func getContainersByLabel(label configsections.Label) (containers []configsections.ContainerConfig, err error) { + pods, err := GetPodsByLabel(label) + if err != nil { + return nil, err + } + for i := range pods.Items { + containers = append(containers, buildContainersFromPodResource(pods.Items[i])...) + } + return containers, nil +} + +// getContainersByLabelByNamespace builds `config.Container`s from containers in pods matching a label. +func getContainersByLabelByNamespace(label configsections.Label, namespace string) (containers []configsections.ContainerConfig, err error) { + pods, err := GetPodsByLabelByNamespace(label, namespace) if err != nil { return nil, err } @@ -88,8 +109,8 @@ func getContainersByLabel(label configsections.Label, namespace string) (contain } // getContainerIdentifiersByLabel builds `config.ContainerIdentifier`s from containers in pods matching a label. -func getContainerIdentifiersByLabel(label configsections.Label, namespace string) (containerIDs []configsections.ContainerIdentifier, err error) { - containers, err := getContainersByLabel(label, namespace) +func getContainerIdentifiersByLabel(label configsections.Label) (containerIDs []configsections.ContainerIdentifier, err error) { + containers, err := getContainersByLabel(label) if err != nil { return nil, err } @@ -99,10 +120,10 @@ func getContainerIdentifiersByLabel(label configsections.Label, namespace string return containerIDs, nil } -// getContainerByLabel returns exactly one container with the given label. If any other number of containers is found +// getContainerByLabelByNamespace returns exactly one container with the given label. If any other number of containers is found // then an error is returned along with an empty `config.Container`. -func getContainerByLabel(label configsections.Label, namespace string) (container configsections.ContainerConfig, err error) { - containers, err := getContainersByLabel(label, namespace) +func getContainerByLabelByNamespace(label configsections.Label, namespace string) (container configsections.ContainerConfig, err error) { + containers, err := getContainersByLabelByNamespace(label, namespace) if err != nil { return container, err } diff --git a/pkg/config/autodiscover/autodiscover_debug.go b/pkg/config/autodiscover/autodiscover_debug.go index b0b267ced..22ae7e780 100644 --- a/pkg/config/autodiscover/autodiscover_debug.go +++ b/pkg/config/autodiscover/autodiscover_debug.go @@ -45,7 +45,7 @@ const ( // using labels and annotations to populate the data, if it's not fully configured func FindDebugPods(tp *configsections.TestPartner) { label := configsections.Label{Name: debugLabelName, Value: debugLabelValue} - pods, err := GetPodsByLabel(label, defaultNamespace) + pods, err := GetPodsByLabelByNamespace(label, defaultNamespace) if err != nil { log.Panic("can't find debug pods") } diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index 16c1556f3..43cca8578 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -38,7 +38,7 @@ func IsMinikube() bool { // using labels and annotations to populate the data, if it's not fully configured func FindTestPartner(tp *configsections.TestPartner, namespace string) { if tp.TestOrchestratorID.ContainerName == "" { - orchestrator, err := getContainerByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: genericLabelName, Value: orchestratorValue}, namespace) + orchestrator, err := getContainerByLabelByNamespace(configsections.Label{Prefix: tnfLabelPrefix, Name: genericLabelName, Value: orchestratorValue}, namespace) if err != nil { log.Errorf("failed to identify a single test orchestrator container: %s", err) return diff --git a/pkg/config/autodiscover/autodiscover_targets.go b/pkg/config/autodiscover/autodiscover_targets.go index 8a0241b36..a9ecb83c4 100644 --- a/pkg/config/autodiscover/autodiscover_targets.go +++ b/pkg/config/autodiscover/autodiscover_targets.go @@ -46,38 +46,52 @@ var ( // FindTestTarget finds test targets from the current state of the cluster, // using labels and annotations, and add them to the `configsections.TestTarget` passed in. +//nolint:funlen func FindTestTarget(labels []configsections.Label, target *configsections.TestTarget, namespaces []string) { - for _, ns := range namespaces { - // find pods by label - for _, l := range labels { - pods, err := GetPodsByLabel(l, ns) - if err == nil { - for i := range pods.Items { - target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(pods.Items[i])) - target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(pods.Items[i])...) + ns := make(map[string]bool) + for _, n := range namespaces { + ns[n] = true + } + for _, l := range labels { + pods, err := GetPodsByLabel(l) + if err == nil { + for _, pod := range pods.Items { + if ns[pod.Metadata.Namespace] { + target.PodsUnderTest = append(target.PodsUnderTest, buildPodUnderTest(pod)) + target.ContainerConfigList = append(target.ContainerConfigList, buildContainersFromPodResource(pod)...) + } else { + target.NonValidPods = append(target.NonValidPods, buildPodUnderTest(pod)) } - } else { - log.Warnf("failed to query by label: %v %v", l, err) } + } else { + log.Warnf("failed to query by label: %v %v", l, err) } - // Containers to exclude from connectivity tests are optional - identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}, ns) - target.ExcludeContainersFromConnectivityTests = append(target.ExcludeContainersFromConnectivityTests, identifiers...) - - if err != nil { - log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) + } + // Containers to exclude from connectivity tests are optional + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{Prefix: tnfLabelPrefix, Name: skipConnectivityTestsLabel, Value: anyLabelValue}) + for _, id := range identifiers { + if ns[id.Namespace] { + target.ExcludeContainersFromConnectivityTests = append(target.ExcludeContainersFromConnectivityTests, id) } - - csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue, ns) - if err != nil { - log.Warnf("an error (%s) occurred when looking for operators by label", err) + } + if err != nil { + log.Warnf("an error (%s) occurred when getting the containers to exclude from connectivity tests. Attempting to continue", err) + } + csvs, err := GetCSVsByLabel(operatorLabelName, anyLabelValue) + if err != nil { + log.Warnf("an error (%s) occurred when looking for operators by label", err) + } + for _, csv := range csvs.Items { + if ns[csv.Metadata.Namespace] { + csv := csv + target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csv)) } - - for i := range csvs.Items { - target.Operators = append(target.Operators, buildOperatorFromCSVResource(&csvs.Items[i])) + } + dps := FindTestDeploymentsByLabel(labels, target) + for _, dp := range dps { + if ns[dp.Namespace] { + target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, dp) } - - target.DeploymentsUnderTest = append(target.DeploymentsUnderTest, FindTestDeployments(labels, target, ns)...) } target.Nodes = GetNodesList() } @@ -125,9 +139,31 @@ func GetNodesList() (nodes map[string]configsections.Node) { return nodes } -// FindTestDeployments uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, +// FindTestDeploymentsByLabel uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, +// currently partner and fs_diff ones. +func FindTestDeploymentsByLabel(targetLabels []configsections.Label, target *configsections.TestTarget) (deployments []configsections.Deployment) { + for _, label := range targetLabels { + deploymentResourceList, err := GetTargetDeploymentsByLabel(label) + if err != nil { + log.Error("Unable to get deployment list Error: ", err) + } else { + for _, deploymentResource := range deploymentResourceList.Items { + deployment := configsections.Deployment{ + Name: deploymentResource.GetName(), + Namespace: deploymentResource.GetNamespace(), + Replicas: deploymentResource.GetReplicas(), + } + + deployments = append(deployments, deployment) + } + } + } + return deployments +} + +// FindTestDeploymentsByLabelByNamespace uses the containers' namespace to get its parent deployment. Filters out non CNF test deployments, // currently partner and fs_diff ones. -func FindTestDeployments(targetLabels []configsections.Label, target *configsections.TestTarget, namespace string) (deployments []configsections.Deployment) { +func FindTestDeploymentsByLabelByNamespace(targetLabels []configsections.Label, target *configsections.TestTarget, namespace string) (deployments []configsections.Deployment) { for _, label := range targetLabels { deploymentResourceList, err := GetTargetDeploymentsByNamespace(namespace, label) if err != nil { diff --git a/pkg/config/autodiscover/csv_info.go b/pkg/config/autodiscover/csv_info.go index 78ae2324a..2eb1da7bb 100644 --- a/pkg/config/autodiscover/csv_info.go +++ b/pkg/config/autodiscover/csv_info.go @@ -67,9 +67,9 @@ func (csv *CSVResource) annotationUnmarshalError(annotationKey string, err error err, annotationKey, csv.Metadata.Namespace, csv.Metadata.Name) } -// GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that +// GetCSVsByLabelByNamespace will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that // label will be returned, regardless of the labels value. -func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { +func GetCSVsByLabelByNamespace(labelName, labelValue, namespace string) (*CSVList, error) { out := executeOcGetCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue}), namespace) log.Debug("JSON output for all pods labeled with: ", labelName) @@ -83,3 +83,20 @@ func GetCSVsByLabel(labelName, labelValue, namespace string) (*CSVList, error) { return &csvList, nil } + +// GetCSVsByLabel will return all CSVs with a given label value. If `labelValue` is an empty string, all CSVs with that +// label will be returned, regardless of the labels value. +func GetCSVsByLabel(labelName, labelValue string) (*CSVList, error) { + out := executeOcGetAllCommand(resourceTypeCSV, buildLabelQuery(configsections.Label{Prefix: tnfLabelPrefix, Name: labelName, Value: labelValue})) + + log.Debug("JSON output for all pods labeled with: ", labelName) + log.Debug("Command: ", out) + + var csvList CSVList + err := jsonUnmarshal([]byte(out), &csvList) + if err != nil { + return nil, err + } + + return &csvList, nil +} diff --git a/pkg/config/autodiscover/deployment_info.go b/pkg/config/autodiscover/deployment_info.go index 8268702bf..dd5eb52f5 100644 --- a/pkg/config/autodiscover/deployment_info.go +++ b/pkg/config/autodiscover/deployment_info.go @@ -113,3 +113,20 @@ func GetTargetDeploymentsByNamespace(namespace string, targetLabel configsection return &deploymentList, nil } + +// GetTargetDeploymentsByLabel will return all deployments that have pods with a given label. +func GetTargetDeploymentsByLabel(targetLabel configsections.Label) (*DeploymentList, error) { + labelQuery := fmt.Sprintf("%q==%q", buildLabelName(targetLabel.Prefix, targetLabel.Name), targetLabel.Value) + jqArgs := fmt.Sprintf("'[.items[] | select(.spec.template.metadata.labels.%s)]'", labelQuery) + ocCmd := fmt.Sprintf("oc get %s -A -o json | jq %s", resourceTypeDeployment, jqArgs) + + out := execCommandOutput(ocCmd) + + var deploymentList DeploymentList + err := jsonUnmarshal([]byte(out), &deploymentList.Items) + if err != nil { + return nil, err + } + + return &deploymentList, nil +} diff --git a/pkg/config/autodiscover/pod_info.go b/pkg/config/autodiscover/pod_info.go index 95cbe4545..961a2ebf7 100644 --- a/pkg/config/autodiscover/pod_info.go +++ b/pkg/config/autodiscover/pod_info.go @@ -153,9 +153,10 @@ func (pr *PodResource) annotationUnmarshalError(annotationKey string, err error) err, annotationKey, pr.Metadata.Namespace, pr.Metadata.Name) } -// GetPodsByLabel will return all pods with a given label value. If `labelValue` is an empty string, all pods with that +// GetPodsByLabelByNamespace will return all pods with a given label value in provided namespace. +// If `labelValue` is an empty string, all pods with that // label will be returned, regardless of the labels value. -func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, error) { +func GetPodsByLabelByNamespace(label configsections.Label, namespace string) (*PodList, error) { out := executeOcGetCommand(resourceTypePods, buildLabelQuery(label), namespace) log.Debug("JSON output for all pods labeled with: ", label) @@ -177,3 +178,29 @@ func GetPodsByLabel(label configsections.Label, namespace string) (*PodList, err podList.Items = pods return &podList, nil } + +// GetPodsByLabelByNamespace will return all pods with a given label value. +// If `labelValue` is an empty string, all pods with that +// label will be returned, regardless of the labels value. +func GetPodsByLabel(label configsections.Label) (*PodList, error) { + out := executeOcGetAllCommand(resourceTypePods, buildLabelQuery(label)) + + log.Debug("JSON output for all pods labeled with: ", label) + log.Debug("Command: ", out) + + var podList PodList + err := jsonUnmarshal([]byte(out), &podList) + if err != nil { + return nil, err + } + + // Filter out terminating pods and pending/unscheduled pods + var pods []*PodResource + for _, pod := range podList.Items { + if pod.Metadata.DeletionTimestamp == "" || pod.Status.Phase != podPhaseRunning { + pods = append(pods, pod) + } + } + podList.Items = pods + return &podList, nil +} diff --git a/pkg/config/configsections/common.go b/pkg/config/configsections/common.go index 7f757c2a8..5b797441b 100644 --- a/pkg/config/configsections/common.go +++ b/pkg/config/configsections/common.go @@ -80,6 +80,9 @@ type TestTarget struct { DeploymentsUnderTest []Deployment `yaml:"deploymentsUnderTest" json:"deploymentsUnderTest"` // PodsUnderTest is the list of the pods that needs to be tested. Each entry is a single pod to be tested. PodsUnderTest []Pod `yaml:"podsUnderTest,omitempty" json:"podsUnderTest,omitempty"` + // NonValidPods contains a list of pods that share the same labels with Pods Under Test + // without belonging to namespaces under test + NonValidPods []Pod // ContainerConfigList is the list of containers that needs to be tested. ContainerConfigList []ContainerConfig `yaml:"containersUnderTest" json:"containersUnderTest"` // ExcludeContainersFromConnectivityTests excludes specific containers from network connectivity tests. This is particularly useful for containers that don't have ping available. From fea1e7c8e4c8c003b018b0631492deed499b4f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Pe=C3=B1a?= Date: Mon, 22 Nov 2021 15:31:15 +0100 Subject: [PATCH 167/344] Update CONTRIBUTING.md to reflect current status (#484) The CONTRIBUTING.md file had an outdated list of core reviewers, and provided a non-working command to run unit tests. Co-authored-by: Jun Chen --- CONTRIBUTING.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8af1b58e5..6ef9cbde3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,16 +6,11 @@ conduct themselves in a professional and respectful manner. ## Peer review -Although this is an open source project, a review is required from one of the following committers prior to merging a -Pull Request: +Although this is an open source project, an approval is required from at least two of the +[CNF Cert team members with write privileges]https://github.com/orgs/test-network-function/teams/cnfcert/members) +prior to merging a Pull Request. -* Ryan Goulding (rgoulding@redhat.com) -* Charlie Wheeler-Robinson (cwheeler@redhat.com) -* David Spence (dspence@redhat.com) - -This list is expected to grow over time. - -*No Self Review is allowed.* Each Pull Request should be peer reviewed prior to merge. +*No Self Review is allowed.* Each Pull Request will be peer reviewed prior to merge. ## Workflow @@ -42,7 +37,7 @@ multiple small commits where possible. As always, you should ensure that tests Request. To run the unit tests issue the following command: ```bash -make unit-tests +make test ``` Changes are more likely to be accepted if they are made up of small and self-contained commits, which leads on to @@ -57,7 +52,7 @@ was made. Commit messages are again something that has been widely written abou here. Contributors should follow [these seven rules](https://chris.beams.io/posts/git-commit/#seven-rules) and keep individual -commits focussed (`git add -p` will help with this). +commits focused (`git add -p` will help with this). ## Test Implementation guidelines @@ -81,7 +76,7 @@ As always, you should ensure that tests should pass prior to submitting a Pull R following command: ```bash -make unit-tests +make test ``` ## Configuration guidelines @@ -108,12 +103,12 @@ that the accompanying documentation and guides are updated to include that infor Ensure `goimports` has been run against all Pull Requests prior to submission. -In addition, te `test-network-function` project committers expect all Pull Requests have no linting errors when the +In addition, the `test-network-function` project committers expect all Pull Requests have no linting errors when the configured linters are used. Please ensure you run `make lint` and resolve any issues in your changes before submitting your PR. Disabled linting must be justified. Finally, all contributions should follow the guidance of [Effective Go](https://golang.org/doc/effective_go.html) -unless there is a clear and considered reason not to. Contribution are more likely to be accepted quickly if any +unless there is a clear and considered reason not to. Contributions are more likely to be accepted quickly if any divergence from the guidelines is justified before someone has to ask about it. ## Mock guidelines From 9cdc18576e7efa318492ef0a20f3c4c8ef81ba36 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Mon, 22 Nov 2021 12:41:53 -0600 Subject: [PATCH 168/344] Add tests for isMaster/isWorker (#483) --- .../configsections/{Node.go => node.go} | 0 pkg/config/configsections/node_test.go | 79 +++++++++++++++++++ 2 files changed, 79 insertions(+) rename pkg/config/configsections/{Node.go => node.go} (100%) create mode 100644 pkg/config/configsections/node_test.go diff --git a/pkg/config/configsections/Node.go b/pkg/config/configsections/node.go similarity index 100% rename from pkg/config/configsections/Node.go rename to pkg/config/configsections/node.go diff --git a/pkg/config/configsections/node_test.go b/pkg/config/configsections/node_test.go new file mode 100644 index 000000000..1d38b9f67 --- /dev/null +++ b/pkg/config/configsections/node_test.go @@ -0,0 +1,79 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package configsections + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsMaster(t *testing.T) { + testCases := []struct { + nodeLabel string + expectedOutput bool + }{ + { + expectedOutput: true, + nodeLabel: MasterLabel, + }, + { + expectedOutput: false, + nodeLabel: WorkerLabel, + }, + { + expectedOutput: false, + nodeLabel: "", + }, + } + + for _, tc := range testCases { + testNode := Node{ + Name: "mastertest", + Labels: []string{tc.nodeLabel}, + } + assert.Equal(t, tc.expectedOutput, testNode.IsMaster()) + } +} + +func TestIsWorker(t *testing.T) { + testCases := []struct { + nodeLabel string + expectedOutput bool + }{ + { + expectedOutput: true, + nodeLabel: WorkerLabel, + }, + { + expectedOutput: false, + nodeLabel: MasterLabel, + }, + { + expectedOutput: false, + nodeLabel: "", + }, + } + + for _, tc := range testCases { + testNode := Node{ + Name: "workertest", + Labels: []string{tc.nodeLabel}, + } + assert.Equal(t, tc.expectedOutput, testNode.IsWorker()) + } +} From deae0b165aa9930a91d8230bc5b274011988b95d Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Mon, 22 Nov 2021 14:30:13 -0500 Subject: [PATCH 169/344] Retiring TNF_MINIKUBE_ONLY (#487) * Retiring TNF_MINIKUBE_ONLY * Rename isMinikube() --- .github/workflows/pre-main.yaml | 1 - .github/workflows/tnf-image.yaml | 1 - README.md | 11 ------ .../autodiscover/autodiscover_partner.go | 4 +- pkg/config/config.go | 2 +- run-cnf-suites.sh | 37 +++++++++++-------- script/run-cfd-container.sh | 2 +- script/run-container.sh | 3 -- test-network-function/common/env.go | 6 +-- test-network-function/diagnostic/suite.go | 6 +-- test-network-function/platform/suite.go | 2 +- 11 files changed, 33 insertions(+), 42 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 3210492b0..e98e2baaa 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -12,7 +12,6 @@ env: IMAGE_NAME: testnetworkfunction/test-network-function IMAGE_TAG: unstable TNF_CONTAINER_CLIENT: docker - TNF_MINIKUBE_ONLY: true TNF_NON_INTRUSIVE_ONLY: false TNF_DISABLE_CONFIG_AUTODISCOVER: false TNF_CONFIG_DIR: /tmp/tnf/config diff --git a/.github/workflows/tnf-image.yaml b/.github/workflows/tnf-image.yaml index e9993bf4f..46878c6c3 100644 --- a/.github/workflows/tnf-image.yaml +++ b/.github/workflows/tnf-image.yaml @@ -21,7 +21,6 @@ env: IMAGE_NAME: testnetworkfunction/test-network-function IMAGE_TAG: latest TNF_CONTAINER_CLIENT: docker - TNF_MINIKUBE_ONLY: true TNF_NON_INTRUSIVE_ONLY: false TNF_DISABLE_CONFIG_AUTODISCOVER: false TNF_CONFIG_DIR: /tmp/tnf/config diff --git a/README.md b/README.md index 9b9696d9c..693bfc4c7 100644 --- a/README.md +++ b/README.md @@ -94,13 +94,6 @@ The `certifiedcontainerinfo` and `certifiedoperatorinfo` sections contain inform to be checked for certification status on Red Hat catalogs. ## Runtime environement variables -### Turn off openshift required tests -When test on CNFs that run on k8s only environment, execute shell command below before compile tool and run test shell script. - -```shell script -export TNF_MINIKUBE_ONLY=true -``` - ### Disable intrusive tests If you would like to skip intrusive tests which may disrupt cluster operations, issue the following: @@ -191,10 +184,6 @@ The autodiscovery first looks for paths in the `$KUBECONFIG` environment variabl See [General tests](#general-tests) for a list of available keywords. -*Note*: The `run-tnf-container.sh` script performs autodiscovery of selected TNF environment variables. -Currently supported environment variables include: -- `TNF_MINIKUBE_ONLY` - ### Running using `docker` instead of `podman` By default, `run-container.sh` utilizes `podman`. However, you can configure an alternate container virtualization diff --git a/pkg/config/autodiscover/autodiscover_partner.go b/pkg/config/autodiscover/autodiscover_partner.go index 43cca8578..21c78f022 100644 --- a/pkg/config/autodiscover/autodiscover_partner.go +++ b/pkg/config/autodiscover/autodiscover_partner.go @@ -29,8 +29,8 @@ const ( orchestratorValue = "orchestrator" ) -func IsMinikube() bool { - minikube, _ := strconv.ParseBool(os.Getenv("TNF_MINIKUBE_ONLY")) +func IsNonOcpCluster() bool { + minikube, _ := strconv.ParseBool(os.Getenv("TNF_NON_OCP_CLUSTER")) return minikube } diff --git a/pkg/config/config.go b/pkg/config/config.go index fd377db7c..d37e208fa 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -356,7 +356,7 @@ func (env *TestEnvironment) discoverNodes() { env.NodesUnderTest = env.createNodes(env.Config.Nodes) env.labelNodes() - if !autodiscover.IsMinikube() { + if !autodiscover.IsNonOcpCluster() { expectedDebugPods := 0 for _, node := range env.NodesUnderTest { if node.HasDebugPod() { diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index b3457ce9e..ed54ce688 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -26,23 +26,23 @@ while [[ $1 == -* ]]; do -h|--help|-\?) usage; exit 0;; -o) if (($# > 1)); then OUTPUT_LOC=$2; shift - else - echo "-o requires an argument" 1>&2 - exit 1 - fi ;; - -s|--skip) - while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do - SKIP="$2|$SKIP" - shift - done;; + else + echo "-o requires an argument" 1>&2 + exit 1 + fi ;; + -s|--skip) + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + SKIP="$2|$SKIP" + shift + done;; -f|--focus) - while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do - FOCUS="$2|$FOCUS" - shift - done;; - -*) echo "invalid option: $1" 1>&2; usage_error;; + while (( "$#" >= 2 )) && ! [[ $2 = --* ]] && ! [[ $2 = -* ]] ; do + FOCUS="$2|$FOCUS" + shift + done;; + -*) echo "invalid option: $1" 1>&2; usage_error;; esac - shift + shift done # specify Junit report file name. GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC --ginkgo.junit-report $OUTPUT_LOC/cnf-certification-tests_junit.xml -ginkgo.v -test.v" @@ -54,6 +54,12 @@ GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC --ginkgo.junit-report $OUT FOCUS=${FOCUS%?} # strip the trailing "|" from the concatenation SKIP=${SKIP%?} # strip the trailing "|" from the concatenation +res=`oc version | grep Server` +if [ -z "$res" ] +then + echo "Minikube or similar detected" + export TNF_NON_OCP_CLUSTER=true +fi # Run cnf-feature-deploy test container if not running inside a container # cgroup file doesn't exist on MacOS. Consider that as not running in container as well if [[ ! -f "/proc/1/cgroup" ]] || grep -q init\.scope /proc/1/cgroup; then @@ -76,4 +82,5 @@ SKIP_STRING="" if [ -n "$SKIP" ]; then SKIP_STRING=-ginkgo.skip="$SKIP" fi + cd ./test-network-function && ./test-network-function.test -ginkgo.focus="$FOCUS" $SKIP_STRING ${GINKGO_ARGS} diff --git a/script/run-cfd-container.sh b/script/run-cfd-container.sh index 10812104b..1d92d7ce3 100755 --- a/script/run-cfd-container.sh +++ b/script/run-cfd-container.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if [ "$TNF_RUN_CFD_TEST" == "true" ] && [ "$TNF_MINIKUBE_ONLY" != "true" ]; then +if [ "$TNF_RUN_CFD_TEST" == "true" ]; then export TNF_IMAGE_NAME=cnf-tests export TNF_IMAGE_TAG="${TNF_CFD_IMAGE_TAG:-4.6}" diff --git a/script/run-container.sh b/script/run-container.sh index b9bd155e7..b7d8c5bc0 100755 --- a/script/run-container.sh +++ b/script/run-container.sh @@ -22,7 +22,6 @@ configure_tnf_container_client CONTAINER_TNF_DIR=/usr/tnf CONTAINER_TNF_KUBECONFIG_FILE_BASE_PATH="$CONTAINER_TNF_DIR/kubeconfig/config" CONTAINER_DEFAULT_NETWORK_MODE=bridge -CONTAINER_DEFAULT_TNF_MINIKUBE_ONLY=false CONTAINER_DEFAULT_TNF_NON_INTRUSIVE_ONLY=true CONTAINER_DEFAULT_TNF_DISABLE_CONFIG_AUTODISCOVER=false LOG_LEVEL_DEFAULT=info @@ -77,7 +76,6 @@ done TNF_IMAGE="${TNF_IMAGE:-$TNF_OFFICIAL_IMAGE}" CONTAINER_NETWORK_MODE="${CONTAINER_NETWORK_MODE:-$CONTAINER_DEFAULT_NETWORK_MODE}" -CONTAINER_TNF_MINIKUBE_ONLY="${TNF_MINIKUBE_ONLY:-$CONTAINER_DEFAULT_TNF_MINIKUBE_ONLY}" CONTAINER_TNF_NON_INTRUSIVE_ONLY="${TNF_NON_INTRUSIVE_ONLY:-$CONTAINER_DEFAULT_TNF_NON_INTRUSIVE_ONLY}" CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER="${TNF_DISABLE_CONFIG_AUTODISCOVER:-$CONTAINER_DEFAULT_TNF_DISABLE_CONFIG_AUTODISCOVER}" LOG_LEVEL="${LOG_LEVEL:-$LOG_LEVEL_DEFAULT}" @@ -104,7 +102,6 @@ ${TNF_CONTAINER_CLIENT} run --rm $DNS_ARG \ $CONFIG_VOLUME_MOUNT_ARG \ -v $OUTPUT_LOC:$CONTAINER_TNF_DIR/claim:Z \ -e KUBECONFIG=$CONTAINER_TNF_KUBECONFIG \ - -e TNF_MINIKUBE_ONLY=$CONTAINER_TNF_MINIKUBE_ONLY \ -e TNF_NON_INTRUSIVE_ONLY=$CONTAINER_TNF_NON_INTRUSIVE_ONLY \ -e TNF_DISABLE_CONFIG_AUTODISCOVER=$CONTAINER_TNF_DISABLE_CONFIG_AUTODISCOVER \ -e TNF_PARTNER_NAMESPACE=$TNF_PARTNER_NAMESPACE \ diff --git a/test-network-function/common/env.go b/test-network-function/common/env.go index 2f1d0f6f8..aa1effc99 100644 --- a/test-network-function/common/env.go +++ b/test-network-function/common/env.go @@ -55,9 +55,9 @@ func GetContext() *interactive.Context { return context } -// IsMinikube returns true when the env var is set, OCP only test would be skipped based on this flag -func IsMinikube() bool { - b, _ := strconv.ParseBool(os.Getenv("TNF_MINIKUBE_ONLY")) +// IsNonOcpCluster returns true when the env var is set, OCP only test would be skipped based on this flag +func IsNonOcpCluster() bool { + b, _ := strconv.ParseBool(os.Getenv("TNF_NON_OCP_CLUSTER")) return b } diff --git a/test-network-function/diagnostic/suite.go b/test-network-function/diagnostic/suite.go index 1ba7a244a..02c8d5fba 100644 --- a/test-network-function/diagnostic/suite.go +++ b/test-network-function/diagnostic/suite.go @@ -214,7 +214,7 @@ func testOcpVersion() { } func testCniPlugins() { - if common.IsMinikube() { + if common.IsNonOcpCluster() { ginkgo.Skip("can't use 'oc debug' in minikube") } // get name of a master node @@ -227,7 +227,7 @@ func testCniPlugins() { } func testNodesHwInfo() { - if common.IsMinikube() { + if common.IsNonOcpCluster() { ginkgo.Skip("can't use 'oc debug' in minikube") } env = config.GetTestEnvironment() @@ -320,7 +320,7 @@ func getNodeLspci(nodeName string) []string { // check CSI driver info in cluster func listClusterCSIInfo() { - if common.IsMinikube() { + if common.IsNonOcpCluster() { ginkgo.Skip("CSI is not checked in minikube") } context := common.GetContext() diff --git a/test-network-function/platform/suite.go b/test-network-function/platform/suite.go index 2b9f6d614..9634989b2 100644 --- a/test-network-function/platform/suite.go +++ b/test-network-function/platform/suite.go @@ -146,7 +146,7 @@ var _ = ginkgo.Describe(common.PlatformAlterationTestKey, func() { }) ginkgo.ReportAfterEach(results.RecordResult) // use this boolean to turn off tests that require OS packages - if !common.IsMinikube() { + if !common.IsNonOcpCluster() { testContainersFsDiff(env) testTainted(env) testHugepages(env) From 46bb545ebee4c9152028232b4a8626e0e9b9a3cd Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 23 Nov 2021 13:57:26 -0600 Subject: [PATCH 170/344] Add more test coverage to autodiscover (#481) --- pkg/config/autodiscover/autodiscover.go | 4 +- .../autodiscover/autodiscover_partner_test.go | 48 +++ pkg/config/autodiscover/autodiscover_test.go | 189 ++++++++++++ .../autodiscover/testdata/testpods_empty.json | 9 + .../testdata/testpods_withlabel.json | 274 ++++++++++++++++++ 5 files changed, 522 insertions(+), 2 deletions(-) create mode 100644 pkg/config/autodiscover/autodiscover_partner_test.go create mode 100644 pkg/config/autodiscover/autodiscover_test.go create mode 100644 pkg/config/autodiscover/testdata/testpods_empty.json create mode 100644 pkg/config/autodiscover/testdata/testpods_withlabel.json diff --git a/pkg/config/autodiscover/autodiscover.go b/pkg/config/autodiscover/autodiscover.go index 403571e01..9b856d0b8 100644 --- a/pkg/config/autodiscover/autodiscover.go +++ b/pkg/config/autodiscover/autodiscover.go @@ -68,7 +68,7 @@ func buildLabelQuery(label configsections.Label) string { return fullLabelName } -func executeOcGetCommand(resourceType, labelQuery, namespace string) string { +var executeOcGetCommand = func(resourceType, labelQuery, namespace string) string { ocCommandToExecute := fmt.Sprintf(ocCommand, resourceType, namespace, labelQuery) match := utils.ExecuteCommand(ocCommandToExecute, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { log.Error("can't run command: ", ocCommandToExecute) @@ -76,7 +76,7 @@ func executeOcGetCommand(resourceType, labelQuery, namespace string) string { return match } -func executeOcGetAllCommand(resourceType, labelQuery string) string { +var executeOcGetAllCommand = func(resourceType, labelQuery string) string { ocCommandToExecute := fmt.Sprintf(ocAllCommand, resourceType, labelQuery) match := utils.ExecuteCommand(ocCommandToExecute, ocCommandTimeOut, interactive.GetContext(expectersVerboseModeEnabled), func() { log.Error("can't run command: ", ocCommandToExecute) diff --git a/pkg/config/autodiscover/autodiscover_partner_test.go b/pkg/config/autodiscover/autodiscover_partner_test.go new file mode 100644 index 000000000..43c27d30d --- /dev/null +++ b/pkg/config/autodiscover/autodiscover_partner_test.go @@ -0,0 +1,48 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsNonOcpCluster(t *testing.T) { + defer os.Unsetenv("TNF_NON_OCP_CLUSTER") + testCases := []struct { + IsNonOcpCluster bool + }{ + { + IsNonOcpCluster: true, + }, + { + IsNonOcpCluster: false, + }, + } + + for _, tc := range testCases { + if tc.IsNonOcpCluster { + os.Setenv("TNF_NON_OCP_CLUSTER", "true") + assert.True(t, IsNonOcpCluster()) + } else { + os.Setenv("TNF_NON_OCP_CLUSTER", "false") + assert.False(t, IsNonOcpCluster()) + } + } +} diff --git a/pkg/config/autodiscover/autodiscover_test.go b/pkg/config/autodiscover/autodiscover_test.go new file mode 100644 index 000000000..bf2992648 --- /dev/null +++ b/pkg/config/autodiscover/autodiscover_test.go @@ -0,0 +1,189 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package autodiscover + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/config/configsections" +) + +func TestBuildLabelQuery(t *testing.T) { + testCases := []struct { + testLabel configsections.Label + expectedOutput string + }{ + { + testLabel: configsections.Label{ + Prefix: "testprefix", + Name: "testname", + Value: "testvalue", + }, + expectedOutput: "testprefix/testname=testvalue", + }, + { + testLabel: configsections.Label{ + Prefix: "testprefix", + Name: "testname", + Value: "", // empty value + }, + expectedOutput: "testprefix/testname", + }, + { + testLabel: configsections.Label{ + Prefix: "", // empty value + Name: "testname", + Value: "testvalue", + }, + expectedOutput: "testname=testvalue", + }, + } + + for _, tc := range testCases { + assert.Equal(t, tc.expectedOutput, buildLabelQuery(tc.testLabel)) + } +} + +func TestGetContainersByLabel(t *testing.T) { + testCases := []struct { + expectedOutput []configsections.ContainerConfig + prefix string + name string + value string + filename string + }{ + { + prefix: "testprefix", + name: "testname", + value: "testvalue", + filename: "testdata/testpods_withlabel.json", + expectedOutput: []configsections.ContainerConfig{ + { + ContainerIdentifier: configsections.ContainerIdentifier{ + Namespace: "kube-system", + PodName: "coredns-78fcd69978-cc94v", + ContainerName: "coredns", + NodeName: "minikube", + }, + }, + }, + }, + { + prefix: "test1", + name: "", + value: "", // no value + filename: "testdata/testpods_empty.json", + expectedOutput: []configsections.ContainerConfig(nil), + }, + } + + origCommand := executeOcGetAllCommand + defer func() { + executeOcGetAllCommand = origCommand + }() + + for _, tc := range testCases { + executeOcGetAllCommand = func(resourceType, labelQuery string) string { + file, _ := os.ReadFile(tc.filename) + return string(file) + } + + containers, _ := getContainersByLabel(configsections.Label{ + Prefix: tc.prefix, + Name: tc.name, + Value: tc.value, + }) + assert.Equal(t, tc.expectedOutput, containers) + } +} + +func TestPerformAutoDiscovery(t *testing.T) { + defer os.Unsetenv(disableAutodiscoverEnvVar) + testCases := []struct { + autoDiscoverEnabled bool + }{ + {autoDiscoverEnabled: true}, + {autoDiscoverEnabled: false}, + } + + for _, tc := range testCases { + if !tc.autoDiscoverEnabled { + os.Setenv(disableAutodiscoverEnvVar, "true") + assert.False(t, PerformAutoDiscovery()) + } else { + os.Setenv(disableAutodiscoverEnvVar, "false") + assert.True(t, PerformAutoDiscovery()) + } + } +} + +//nolint:funlen +func TestGetContainerIdentifiersByLabel(t *testing.T) { + testCases := []struct { + expectedOutput []configsections.ContainerIdentifier + prefix string + name string + value string + filename string + }{ + { + expectedOutput: []configsections.ContainerIdentifier{ + { + Namespace: "kube-system", + PodName: "coredns-78fcd69978-cc94v", + ContainerName: "coredns", + NodeName: "minikube", + }, + }, + prefix: "testprefix", + name: "testname", + value: "testvalue", + filename: "testdata/testpods_withlabel.json", + }, + + { + prefix: "test1", + name: "", + value: "", // no value + filename: "testdata/testpods_empty.json", + expectedOutput: []configsections.ContainerIdentifier(nil), + }, + } + + origCommand := executeOcGetAllCommand + defer func() { + executeOcGetAllCommand = origCommand + }() + + for _, tc := range testCases { + executeOcGetAllCommand = func(resourceType, labelQuery string) string { + file, _ := os.ReadFile(tc.filename) + return string(file) + } + + identifiers, err := getContainerIdentifiersByLabel(configsections.Label{ + Prefix: tc.prefix, + Name: tc.name, + Value: tc.value, + }) + + assert.Nil(t, err) + assert.Equal(t, tc.expectedOutput, identifiers) + } +} diff --git a/pkg/config/autodiscover/testdata/testpods_empty.json b/pkg/config/autodiscover/testdata/testpods_empty.json new file mode 100644 index 000000000..1c318dacd --- /dev/null +++ b/pkg/config/autodiscover/testdata/testpods_empty.json @@ -0,0 +1,9 @@ +{ + "apiVersion": "v1", + "items": [], + "kind": "List", + "metadata": { + "resourceVersion": "", + "selfLink": "" + } +} diff --git a/pkg/config/autodiscover/testdata/testpods_withlabel.json b/pkg/config/autodiscover/testdata/testpods_withlabel.json new file mode 100644 index 000000000..308774a01 --- /dev/null +++ b/pkg/config/autodiscover/testdata/testpods_withlabel.json @@ -0,0 +1,274 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2021-11-16T21:42:18Z", + "generateName": "coredns-78fcd69978-", + "labels": { + "k8s-app": "kube-dns", + "pod-template-hash": "78fcd69978", + "testprefix/testname": "testvalue" + }, + "name": "coredns-78fcd69978-cc94v", + "namespace": "kube-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "coredns-78fcd69978", + "uid": "527d93cd-fb69-4b85-9eff-a044f18b64fb" + } + ], + "resourceVersion": "1069", + "uid": "9365d337-041e-445c-a1e8-77e218bb96a1" + }, + "spec": { + "containers": [ + { + "args": [ + "-conf", + "/etc/coredns/Corefile" + ], + "image": "k8s.gcr.io/coredns/coredns:v1.8.4", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 5, + "httpGet": { + "path": "/health", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "name": "coredns", + "ports": [ + { + "containerPort": 53, + "name": "dns", + "protocol": "UDP" + }, + { + "containerPort": 53, + "name": "dns-tcp", + "protocol": "TCP" + }, + { + "containerPort": 9153, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/ready", + "port": 8181, + "scheme": "HTTP" + }, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "170Mi" + }, + "requests": { + "cpu": "100m", + "memory": "70Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "add": [ + "NET_BIND_SERVICE" + ], + "drop": [ + "all" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/etc/coredns", + "name": "config-volume", + "readOnly": true + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cb6f4", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "Default", + "enableServiceLinks": true, + "nodeName": "minikube", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 2000000000, + "priorityClassName": "system-cluster-critical", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "coredns", + "serviceAccountName": "coredns", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/master" + }, + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/control-plane" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "Corefile", + "path": "Corefile" + } + ], + "name": "coredns" + }, + "name": "config-volume" + }, + { + "name": "kube-api-access-cb6f4", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2021-11-16T21:42:36Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-11-16T21:42:39Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-11-16T21:42:39Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-11-16T21:42:36Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://cf794b9e8c2448815b8b5a47b354c9bf9414a04f6fa567ac3b059851ed6757ab", + "image": "k8s.gcr.io/coredns/coredns:v1.8.4", + "imageID": "docker-pullable://k8s.gcr.io/coredns/coredns@sha256:6e5a02c21641597998b4be7cb5eb1e7b02c0d8d23cce4dd09f4682d463798890", + "lastState": {}, + "name": "coredns", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2021-11-16T21:42:37Z" + } + } + } + ], + "hostIP": "192.168.49.2", + "phase": "Running", + "podIP": "10.244.0.2", + "podIPs": [ + { + "ip": "10.244.0.2" + } + ], + "qosClass": "Burstable", + "startTime": "2021-11-16T21:42:36Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "", + "selfLink": "" + } +} From 132da5f99ff4879d98dd97c44aa1ea09e8b33775 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Tue, 23 Nov 2021 16:53:56 -0500 Subject: [PATCH 171/344] Update links to files from the partner repo (#489) --- CATALOG.md | 2 +- README.md | 2 +- test-network-function/identifiers/identifiers.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 17fe6ac77..630399fa3 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -173,7 +173,7 @@ Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/site Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. +Description|http://test-network-function.com/testcases/networking/icmpv4-connectivity checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the [CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner-deployment.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. Result Type|normative Suggested Remediation|Ensure that the CNF is able to communicate via the Default OpenShift network. In some rare cases, CNFs may require routing table changes in order to communicate over the Default network. In other cases, if the Container base image does not provide the "ip" or "ping" binaries, this test may not be applicable. For instructions on how to exclude a particular container from ICMPv4 connectivity tests, consult: [README.md](https://github.com/test-network-function/test-network-function#issue-161-some-containers-under-test-do-not-contain-ping-or-ip-binary-utilities). Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 diff --git a/README.md b/README.md index 693bfc4c7..d0b71e043 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ the subscription for this CSV. If unset, the CSV name will be used. ### testPartner -This section can also be discovered automatically and should be left commented out unless the partner pods are modified from the original version in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/local-test-infra/) +This section can also be discovered automatically and should be left commented out unless the partner pods are modified from the original version in [cnf-certification-test-partner](https://github.com/test-network-function/cnf-certification-test-partner/tree/main/test-partner) ### certifiedcontainerinfo and certifiedoperatorinfo diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index 80158e575..dbbb63951 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -313,7 +313,7 @@ how to exclude a particular container from ICMPv4 connectivity tests, consult: Description: formDescription(TestICMPv4ConnectivityIdentifier, `checks that each CNF Container is able to communicate via ICMPv4 on the Default OpenShift network. This test case requires the Deployment of the -[CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner.yaml). +[CNF Certification Test Partner](https://github.com/test-network-function/cnf-certification-test-partner/blob/main/test-partner/partner-deployment.yaml). The test ensures that all CNF containers respond to ICMPv4 requests from the Partner Pod, and vice-versa. `), BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", From 4052810df18052b8a8493dcdd5d8aec3221152ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Pe=C3=B1a?= Date: Thu, 25 Nov 2021 18:18:51 +0100 Subject: [PATCH 172/344] Archive test run artifacts (#491) We want to store the test run artifacts for both the smoke and container tests, so we can analyze them after a CI job. --- .github/workflows/pre-main.yaml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index e98e2baaa..42c815a78 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -48,7 +48,7 @@ jobs: - name: make lint run: make lint - + - name: make vet run: make vet @@ -144,6 +144,16 @@ jobs: - name: 'Test: Run test suites' run: ./run-cnf-suites.sh --focus access-control lifecycle platform observablility networking affiliated-certification operator + + - name: Upload smoke test results as an artifact + uses: actions/upload-artifact@v2 + if: always() + with: + name: smoke-tests + path: | + test-network-function/*.xml + test-network-function/claim.json + # Perform smoke tests using a TNF container. - name: Build the `test-network-function` image @@ -170,7 +180,16 @@ jobs: - name: 'Test: Run generic test suite in a TNF container' run: ./run-tnf-container.sh ${{ env.TESTING_CMD_PARAMS }} -f access-control lifecycle platform observablility networking affiliated-certification operator - + + - name: Upload container test results as an artifact + uses: actions/upload-artifact@v2 + if: always() + with: + name: smoke-tests-container + path: | + ${{ env.TNF_OUTPUT_DIR }}/*.xml + ${{ env.TNF_OUTPUT_DIR }}/claim.json + # Push the new unstable TNF image to Quay.io. - name: (if on main and upstream) Authenticate against Quay.io if: ${{ github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }} From 411c3f3ddec7d1389af93ddf5af8fdd7cbe75320 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 25 Nov 2021 19:28:56 +0100 Subject: [PATCH 173/344] Namespaces TC extended with CRs namespace check. (#468) * Namespaces TC extended with CRs namespace check. * Refactor to solve linting. * Remove new line char escaping. * Addressing Jun's comments. - Use .spec.names.plural instead of .spec.names.kind because some resources cannot be listed using the kind name. - Added default and openshift- prefixes to the invalid namespaces list. * Changed CRs check and added NamespacesUnderTest check. * Updated pods namespace check. * Improved failed TC log messages. * README.md file updated. * Addressing Salah's comments. Co-authored-by: Jun Chen --- CATALOG.md | 6 +- test-network-function/accesscontrol/suite.go | 137 +++++++++++++++++- .../identifiers/identifiers.go | 12 +- 3 files changed, 139 insertions(+), 16 deletions(-) diff --git a/CATALOG.md b/CATALOG.md index 630399fa3..762af41e6 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -29,10 +29,10 @@ Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/site Property|Description ---|--- Version|v1.0.0 -Description|http://test-network-function.com/testcases/access-control/namespace tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace. +Description|http://test-network-function.com/testcases/access-control/namespace tests that all CNF's resources (PUTs and CRs) belong to valid namespaces. A valid namespace meets the following conditions: (1) It was declared in the yaml config file under the targetNameSpaces tag. (2) It doesn't have any of the following prefixes: default, openshift-, istio- and aspenmesh- Result Type|normative -Suggested Remediation|Ensure that your CNF utilizes a CNF-specific namespace. Additionally, the CNF-specific namespace should not start with "openshift-", except in rare cases. -Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 +Suggested Remediation|Ensure that your CNF utilizes namespaces declared in the yaml config file. Additionally, the namespaces should not start with "default, openshift-, istio- or aspenmesh-", except in rare cases. +Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2, 16.3.8 & 16.3.9 ### http://test-network-function.com/testcases/access-control/pod-role-bindings Property|Description diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index fb3716b91..8ed8c691d 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -31,11 +31,37 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf/handlers/serviceaccount" "github.com/test-network-function/test-network-function/pkg/tnf/reel" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/pkg/utils" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" "github.com/test-network-function/test-network-function/test-network-function/results" ) +const ( + // ocGetCrPluralNameFormat is the CR name to use with "oc get ". + ocGetCrPluralNameFormat = "oc get crd %s -o jsonpath='{.spec.names.plural}'" + + // ocGetCrNamespaceFormat is the "oc get" format string to get the namespaced-only resources created for a given CRD. + ocGetCrNamespaceFormat = "oc get %s -A -o go-template='{{range .items}}{{if .metadata.namespace}}{{.metadata.name}},{{.metadata.namespace}}{{\"\n\"}}{{end}}{{end}}'" +) + +var ( + invalidNamespacePrefixes = []string{ + "default", + "openshift-", + "istio-", + "aspenmesh-", + } + + tcClaimLogPrintf = func(format string, args ...interface{}) { + message := fmt.Sprintf(format+"\n", args...) + _, err := ginkgo.GinkgoWriter.Write([]byte(message)) + if err != nil { + log.Errorf("Ginkgo writer could not write msg '%s' because: %s", message, err) + } + } +) + var _ = ginkgo.Describe(common.AccessControlTestKey, func() { conf, _ := ginkgo.GinkgoConfiguration() if testcases.IsInFocus(conf.FocusStrings, common.AccessControlTestKey) { @@ -130,16 +156,113 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, }) } +func getCrsNamespaces(crdName, crdKind string) (map[string]string, error) { + const expectedNumFields = 2 + const crNameFieldIdx = 0 + const namespaceFieldIdx = 0 + + gomega.Expect(crdKind).NotTo(gomega.BeEmpty()) + getCrNamespaceCommand := fmt.Sprintf(ocGetCrNamespaceFormat, crdKind) + cmdOut := utils.ExecuteCommand(getCrNamespaceCommand, common.DefaultTimeout, common.GetContext(), func() { + tcClaimLogPrintf("CRD %s: Failed to get CRs (kind=%s)", crdName, crdKind) + }) + + crNamespaces := map[string]string{} + + if cmdOut == "" { + // Filter out empty (0 CRs) output. + return crNamespaces, nil + } + + lines := strings.Split(cmdOut, "\n") + for _, line := range lines { + lineFields := strings.Split(line, ",") + if len(lineFields) != expectedNumFields { + return crNamespaces, fmt.Errorf("failed to parse output line %s", line) + } + crNamespaces[lineFields[crNameFieldIdx]] = lineFields[namespaceFieldIdx] + } + + return crNamespaces, nil +} + +func testCrsNamespaces(crNames, configNamespaces []string) (invalidCrs map[string][]string) { + invalidCrs = map[string][]string{} + for _, crdName := range crNames { + getCrPluralNameCommand := fmt.Sprintf(ocGetCrPluralNameFormat, crdName) + crdPluralName := utils.ExecuteCommand(getCrPluralNameCommand, common.DefaultTimeout, common.GetContext(), func() { + tcClaimLogPrintf("CRD %s: Failed to get CR plural name.", crdName) + }) + + crNamespaces, err := getCrsNamespaces(crdName, crdPluralName) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Failed to get CRs for CRD %s - Error: %v", crdName, err)) + } + + ginkgo.By(fmt.Sprintf("CRD %s has %d CRs (plural name: %s).", crdName, len(crNamespaces), crdPluralName)) + for crName, namespace := range crNamespaces { + ginkgo.By(fmt.Sprintf("Checking CR %s - Namespace %s", crName, namespace)) + found := false + for _, configNamespace := range configNamespaces { + if namespace == configNamespace { + found = true + break + } + } + + if !found { + tcClaimLogPrintf("CRD: %s (kind:%s) - CR %s has an invalid namespace (%s)", crdName, crdPluralName, crName, namespace) + if crNames, exists := invalidCrs[crdName]; exists { + invalidCrs[crdName] = append(crNames, crName) + } else { + invalidCrs[crdName] = []string{crName} + } + } + } + } + return invalidCrs +} + func testNamespace(env *config.TestEnvironment) { - ginkgo.When("test deployment namespace", func() { + ginkgo.When("test CNF namespaces", func() { testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestNamespaceBestPracticesIdentifier) ginkgo.It(testID, func() { - for _, podUnderTest := range env.PodsUnderTest { - podName := podUnderTest.Name - podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s, should not be 'default' or begin with openshift-", podNamespace, podName)) - gomega.Expect(podNamespace).To(gomega.Not(gomega.Equal("default"))) - gomega.Expect(podNamespace).To(gomega.Not(gomega.HavePrefix("openshift-"))) + ginkgo.By(fmt.Sprintf("CNF resources' namespaces should not have any of the following prefixes: %v", invalidNamespacePrefixes)) + var failedNamespaces []string + for _, namespace := range env.NameSpacesUnderTest { + ginkgo.By(fmt.Sprintf("Checking namespace %s", namespace)) + for _, invalidPrefix := range invalidNamespacePrefixes { + if strings.HasPrefix(namespace, invalidPrefix) { + tcClaimLogPrintf("Namespace %s has invalid prefix %s", namespace, invalidPrefix) + failedNamespaces = append(failedNamespaces, namespace) + } + } + } + + if failedNamespacesNum := len(failedNamespaces); failedNamespacesNum > 0 { + ginkgo.Fail(fmt.Sprintf("Found %d namespaces with an invalid prefix.", failedNamespacesNum)) + } + + ginkgo.By(fmt.Sprintf("CNF pods' should belong to any of the configured namespaces: %v", env.NameSpacesUnderTest)) + + if nonValidPodsNum := len(env.Config.NonValidPods); nonValidPodsNum > 0 { + for _, invalidPod := range env.Config.NonValidPods { + tcClaimLogPrintf("Pod %s has invalid namespace %s", invalidPod.Name, invalidPod.Namespace) + } + + ginkgo.Fail(fmt.Sprintf("Found %d pods under test belonging to invalid namespaces.", nonValidPodsNum)) + } + + ginkgo.By(fmt.Sprintf("CRs from autodiscovered CRDs should belong to the configured namespaces: %v", env.NameSpacesUnderTest)) + invalidCrs := testCrsNamespaces(env.CrdNames, env.NameSpacesUnderTest) + + if invalidCrsNum := len(invalidCrs); invalidCrsNum > 0 { + for crdName, crs := range invalidCrs { + for _, crName := range crs { + tcClaimLogPrintf("CRD %s - CR %s has an invalid namespace.", crdName, crName) + } + } + ginkgo.Fail(fmt.Sprintf("Found %d CRs belonging to invalid namespaces.", invalidCrsNum)) } }) }) diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index dbbb63951..f50116b89 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -322,13 +322,13 @@ The test ensures that all CNF containers respond to ICMPv4 requests from the Par TestNamespaceBestPracticesIdentifier: { Identifier: TestNamespaceBestPracticesIdentifier, Type: normativeResult, - Remediation: `Ensure that your CNF utilizes a CNF-specific namespace. Additionally, the CNF-specific namespace -should not start with "openshift-", except in rare cases.`, + Remediation: `Ensure that your CNF utilizes namespaces declared in the yaml config file. Additionally, +the namespaces should not start with "default, openshift-, istio- or aspenmesh-", except in rare cases.`, Description: formDescription(TestNamespaceBestPracticesIdentifier, - `tests that CNFs utilize a CNF-specific namespace, and that the namespace does not start with "openshift-". -OpenShift may host a variety of CNF and software applications, and multi-tenancy of such applications is supported -through namespaces. As such, each CNF should be a good neighbor, and utilize an appropriate, unique namespace.`), - BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2", + `tests that all CNF's resources (PUTs and CRs) belong to valid namespaces. A valid namespace meets +the following conditions: (1) It was declared in the yaml config file under the targetNameSpaces +tag. (2) It doesn't have any of the following prefixes: default, openshift-, istio- and aspenmesh-`), + BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.2, 16.3.8 & 16.3.9", }, TestNonDefaultGracePeriodIdentifier: { From 1b281ffc88ba6997cc71e22f45d0db939ca0945e Mon Sep 17 00:00:00 2001 From: aabughosh <88486034+aabughosh@users.noreply.github.com> Date: Fri, 26 Nov 2021 16:29:00 +0200 Subject: [PATCH 174/344] CNFCER-185: add handler to check image pull policy (#488) * CNFCER-185: add handler to check image pull policy * CNFCER-185: test image pull test * CNFCER-185: update package name * CNFCER-185: few update for linet * CNFCER-185: few update for linet * CNFCER-185: update IfNotPresent * CNFCER-185: update IfNotPresent * added the test as json * update liner * update liner * update the test * update the test * update the test * update the test * update the test * update the test * update the test * update the test * update the test * remove the empty from test * update the identifier * update the catalog * update the catalog * update the catalog * update the catalog * update the loop to go into the pod under test --- CATALOG.md | 19 +++ pkg/tnf/handlers/imagepullpolicy/doc.go | 17 +++ .../imagepullpolicy/imagepullpolicy.json | 32 ++++ .../imagepullpolicy/imagepullpolicy_test.go | 138 ++++++++++++++++++ pkg/tnf/identifier/identifiers.go | 18 +++ .../identifiers/identifiers.go | 14 ++ test-network-function/lifecycle/suite.go | 34 +++++ 7 files changed, 272 insertions(+) create mode 100644 pkg/tnf/handlers/imagepullpolicy/doc.go create mode 100644 pkg/tnf/handlers/imagepullpolicy/imagepullpolicy.json create mode 100644 pkg/tnf/handlers/imagepullpolicy/imagepullpolicy_test.go diff --git a/CATALOG.md b/CATALOG.md index 762af41e6..78fc96fb6 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -114,6 +114,15 @@ Description|http://test-network-function.com/testcases/lifecycle/container-shutd Result Type|normative Suggested Remediation| It's considered best-practices to define prestop for proper management of container lifecycle. The prestop can be used to gracefully stop the container and clean resources (e.g., DB connection). The prestop can be configured using : 1) Exec : executes the supplied command inside the container 2) HTTP : executes HTTP request against the specified endpoint. When defined. K8s will handle shutdown of the container using the following: 1) K8s first execute the preStop hook inside the container. 2) K8s will wait for a grace period. 3) K8s will clean the remaining processes using KILL signal. Best Practice Reference|[CNF Best Practice V1.2](https://connect.redhat.com/sites/default/files/2021-03/Cloud%20Native%20Network%20Function%20Requirements.pdf) Section 6.2 +### http://test-network-function.com/testcases/lifecycle/image-pull-policy + +Property|Description +---|--- +Version|v1.0.0 +Description|http://test-network-function.com/testcases/lifecycle/image-pull-policy Ensure that the containers under test are using IfNotPresent as Image Pull Policy.. +Result Type|normative +Suggested Remediation|Ensure that the containers under test are using IfNotPresent as Image Pull Policy. +Best Practice Reference|https://docs.google.com/document/d/1wRHMk1ZYUSVmgp_4kxvqjVOKwolsZ5hDXjr5MLy-wbg/edit# Section 15.6 ### http://test-network-function.com/testcases/lifecycle/pod-high-availability Property|Description @@ -460,6 +469,16 @@ Intrusive|false Modifications Persist After Test|false Runtime Binaries Required|`grep`, `cut`, `oc`, `grep` +### http://test-network-function.com/tests/imagepullpolicy +Property|Description +---|--- +Version|v1.0.0 +Description|A generic test used to get Image Pull Policy type. +Result Type|normative +Intrusive|false +Modifications Persist After Test|false +Runtime Binaries Required|`oc` + ### http://test-network-function.com/tests/ipaddr Property|Description ---|--- diff --git a/pkg/tnf/handlers/imagepullpolicy/doc.go b/pkg/tnf/handlers/imagepullpolicy/doc.go new file mode 100644 index 000000000..f9af0628f --- /dev/null +++ b/pkg/tnf/handlers/imagepullpolicy/doc.go @@ -0,0 +1,17 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +// Package imagepullpolicy provides a test for imagepullpolicy. +package imagepullpolicy diff --git a/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy.json b/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy.json new file mode 100644 index 000000000..2f8c9c56d --- /dev/null +++ b/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy.json @@ -0,0 +1,32 @@ +{ + "identifier" : { + "url" : "http://test-network-function.com/tests/imagepullpolicy", + "version": "v1.0.0" + }, + "description": "A generic test used to get Image Pull Policy type", + "testResult": 0, + "testTimeout": 5000000000, + "reelFirstStep": { + "execute": "oc get pod {{.POD_NAME}} -n {{.POD_NAMESPACE}} -o json | jq -r '.spec.containers[{{.CONTAINER_NUM}}].imagePullPolicy'", + "expect":["(?m)IfNotPresent", + "(?m)Always", + "(?m)Never", + "(?)"], + "timeout": 5000000000 + }, + "resultContexts":[ + { + "pattern": "(?m)IfNotPresent", + "defaultResult": 1 + }, + { + "pattern": "(?m)Always", + "defaultResult": 2 + }, + { + "pattern": "(?m)Never", + "defaultResult": 2 + } + ] + } + \ No newline at end of file diff --git a/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy_test.go b/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy_test.go new file mode 100644 index 000000000..2e7c2e9d6 --- /dev/null +++ b/pkg/tnf/handlers/imagepullpolicy/imagepullpolicy_test.go @@ -0,0 +1,138 @@ +// Copyright (C) 2020-2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package imagepullpolicy_test + +import ( + "fmt" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/tnf/identifier" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" + "github.com/xeipuuv/gojsonschema" +) + +const ( + testTimeoutDuration = time.Second * 5 +) + +var ( + genericTestSchemaFile = path.Join("schemas", "generic-test.schema.json") + imagepullFilename = "imagepullpolicy.json" + /* #nosec G101 */ + expectedPassPattern = "(?m)IfNotPresent" + expectedFailPattern = "(?m)Always" + pathRelativeToRoot = path.Join("..", "..", "..", "..") + pathToTestSchemaFile = path.Join(pathRelativeToRoot, genericTestSchemaFile) + testPodNameSpace = "testnamespace" + testPodName = "testPodname" + testContainerNum = 0 + testInputSuccess = "IfNotPresent" + testInputFilure = "Always" +) + +func createTest() (*tnf.Tester, []reel.Handler, *gojsonschema.Result, error) { + values := make(map[string]interface{}) + values["POD_NAMESPACE"] = testPodNameSpace + values["POD_NAME"] = testPodName + values["CONTAINER_NUM"] = testContainerNum + return generic.NewGenericFromMap(imagepullFilename, pathToTestSchemaFile, values) +} + +func TestImagePullPolicy_Args(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Nil(t, (*test).Args()) +} + +func TestImagePullPolicy_GetIdentifier(t *testing.T) { + test, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, identifier.ImagePullPolicyIdentifier, (*test).GetIdentifier()) +} + +func TestImagePullPolicy_ReelFirst(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + step := handler.ReelFirst() + expectedCommand := fmt.Sprintf("oc get pod %s -n %s -o json | jq -r '.spec.containers[%d].imagePullPolicy'", testPodName, testPodNameSpace, testContainerNum) + assert.Equal(t, expectedCommand, step.Execute) + assert.Contains(t, step.Expect, expectedPassPattern, expectedFailPattern) + assert.Equal(t, testTimeoutDuration, step.Timeout) +} + +func TestImagePullPolicy_ReelEof(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + // just ensure there isn't a panic + handler.ReelEOF() +} + +func TestImagePullPolicy_ReelTimeout(t *testing.T) { + _, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + assert.Nil(t, handler.ReelTimeout()) +} + +func TestImagePullPolicy_ReelMatch(t *testing.T) { + tester, handlers, jsonParseResult, err := createTest() + + assert.Nil(t, err) + assert.True(t, jsonParseResult.Valid()) + assert.NotNil(t, handlers) + + assert.Equal(t, 1, len(handlers)) + handler := handlers[0] + step := handler.ReelMatch(expectedPassPattern, "", testInputSuccess) + + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, (*tester).Result()) + step = handler.ReelMatch(expectedFailPattern, "", testInputFilure) + + assert.Nil(t, step) + assert.Equal(t, tnf.FAILURE, (*tester).Result()) +} diff --git a/pkg/tnf/identifier/identifiers.go b/pkg/tnf/identifier/identifiers.go index 8c9a42657..ea7764399 100644 --- a/pkg/tnf/identifier/identifiers.go +++ b/pkg/tnf/identifier/identifiers.go @@ -33,6 +33,7 @@ const ( roleBindingIdentifierURL = "http://test-network-function.com/tests/rolebinding" clusterRoleBindingIdentifierURL = "http://test-network-function.com/tests/clusterrolebinding" nodePortIdentifierURL = "http://test-network-function.com/tests/nodeport" + ImagePullPolicyIdentifierURL = "http://test-network-function.com/tests/imagepullpolicy" nodeNamesIdentifierURL = "http://test-network-function.com/tests/nodenames" nodeTaintedIdentifierURL = "http://test-network-function.com/tests/nodetainted" gracePeriodIdentifierURL = "http://test-network-function.com/tests/gracePeriod" @@ -278,6 +279,18 @@ var Catalog = map[string]TestCatalogEntry{ dependencies.GrepBinaryName, }, }, + ImagePullPolicyIdentifierURL: { + Identifier: ImagePullPolicyIdentifier, + Description: "A generic test used to get Image Pull Policy type.", + Type: Normative, + IntrusionSettings: IntrusionSettings{ + ModifiesSystem: false, + ModificationIsPersistent: false, + }, + BinaryDependencies: []string{ + dependencies.OcBinaryName, + }, + }, nodeNamesIdentifierURL: { Identifier: NodeNamesIdentifier, Description: "A generic test used to get node names", @@ -734,6 +747,11 @@ var NodePortIdentifier = Identifier{ SemanticVersion: versionOne, } +var ImagePullPolicyIdentifier = Identifier{ + URL: ImagePullPolicyIdentifierURL, + SemanticVersion: versionOne, +} + // NodeNamesIdentifier is the Identifier used to represent the generic NodeNames test. var NodeNamesIdentifier = Identifier{ URL: nodeNamesIdentifierURL, diff --git a/test-network-function/identifiers/identifiers.go b/test-network-function/identifiers/identifiers.go index f50116b89..5a3dccb95 100644 --- a/test-network-function/identifiers/identifiers.go +++ b/test-network-function/identifiers/identifiers.go @@ -31,6 +31,7 @@ const ( normativeResult = "normative" url = "http://test-network-function.com/testcases" versionOne = "v1.0.0" + bestPracticeDocV1dot3URL = "https://docs.google.com/document/d/1wRHMk1ZYUSVmgp_4kxvqjVOKwolsZ5hDXjr5MLy-wbg/edit#" ) // TestCaseDescription describes a JUnit test case. @@ -147,6 +148,11 @@ var ( Url: formTestURL(common.LifecycleTestKey, "pod-owner-type"), Version: versionOne, } + // TestImagePullPolicyIdentifier ensures represent image pull policy practices. + TestImagePullPolicyIdentifier = claim.Identifier{ + Url: formTestURL(common.LifecycleTestKey, "image-pull-policy"), + Version: versionOne, + } // TestPodRecreationIdentifier ensures recreation best practices. TestPodRecreationIdentifier = claim.Identifier{ Url: formTestURL(common.LifecycleTestKey, "pod-recreation"), @@ -427,6 +433,14 @@ ClusterRoleBindings, if possible.`, `tests that CNF Pod(s) are deployed as part of a ReplicaSet(s)/StatefulSet(s).`), BestPracticeReference: bestPracticeDocV1dot2URL + " Section 6.3.3 and 6.3.8", }, + TestImagePullPolicyIdentifier: { + Identifier: TestImagePullPolicyIdentifier, + Type: normativeResult, + Remediation: `Ensure that the containers under test are using IfNotPresent as Image Pull Policy.`, + Description: formDescription(TestImagePullPolicyIdentifier, + `Ensure that the containers under test are using IfNotPresent as Image Pull Policy..`), + BestPracticeReference: bestPracticeDocV1dot3URL + " Section 15.6", + }, TestPodRoleBindingsBestPracticesIdentifier: { Identifier: TestPodRoleBindingsBestPracticesIdentifier, diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index bddef1622..adcdc2b6e 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -75,6 +75,10 @@ var ( // relativePodTestPath is the relative path to the podantiaffinity.json test case. relativePodTestPath = path.Join(common.PathRelativeToRoot, podAntiAffinityTestPath) + + // relativeimagepullpolicyTestPath is the relative path to the imagepullpolicy.json test case. + imagepullpolicyTestPath = path.Join("pkg", "tnf", "handlers", "imagepullpolicy", "imagepullpolicy.json") + relativeimagepullpolicyTestPath = path.Join(common.PathRelativeToRoot, imagepullpolicyTestPath) ) var drainTimeout = time.Duration(drainTimeoutMinutes) * time.Minute @@ -95,6 +99,8 @@ var _ = ginkgo.Describe(common.LifecycleTestKey, func() { ginkgo.ReportAfterEach(results.RecordResult) + testImagePolicy(env) + testNodeSelector(env) testGracePeriod(env) @@ -479,3 +485,31 @@ func testOwner(env *config.TestEnvironment) { } }) } + +func testImagePolicy(env *config.TestEnvironment) { + testID := identifiers.XformToGinkgoItIdentifier(identifiers.TestImagePullPolicyIdentifier) + ginkgo.It(testID, func() { + context := common.GetContext() + for _, podUnderTest := range env.PodsUnderTest { + values := make(map[string]interface{}) + ContainerCount := podUnderTest.ContainerCount + values["POD_NAMESPACE"] = podUnderTest.Namespace + values["POD_NAME"] = podUnderTest.Name + for i := 0; i < ContainerCount; i++ { + values["CONTAINER_NUM"] = i + tester, handlers, result, err := generic.NewGenericFromMap(relativeimagepullpolicyTestPath, common.RelativeSchemaPath, values) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(test).ToNot(gomega.BeNil()) + + test.RunAndValidate() + } + } + }) +} From 8426219758c10af66ae3ae4009b63b96c7106454 Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Fri, 26 Nov 2021 12:40:58 -0500 Subject: [PATCH 175/344] Utility function for creating generic test handler (#492) * Utility function for creating generic test handler * Update suite.go Co-authored-by: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> --- pkg/utils/utils.go | 25 +++++++++----- test-network-function/lifecycle/suite.go | 35 +++----------------- test-network-function/observability/suite.go | 18 ++-------- test-network-function/operator/suite.go | 14 +++----- 4 files changed, 28 insertions(+), 64 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 765ab65c4..384f09f48 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -14,6 +14,7 @@ import ( "github.com/test-network-function/test-network-function/pkg/tnf" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/interactive" + "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) var ( @@ -88,15 +89,8 @@ func ExecuteCommand(command string, timeout time.Duration, context *interactive. log.Debugf("Command handler's COMMAND string value: %s", values["COMMAND"]) - tester, handler, result, err := generic.NewGenericFromMap(commandHandlerFilePath, handlerJSONSchemaFilePath, values) - - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handler).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - - test, err := tnf.NewTest(context.GetExpecter(), *tester, handler, context.GetErrorChannel()) + tester, handlers := NewGenericTestAndValidate(commandHandlerFilePath, handlerJSONSchemaFilePath, values) + test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(tester).ToNot(gomega.BeNil()) @@ -110,3 +104,16 @@ func ExecuteCommand(command string, timeout time.Duration, context *interactive. match := genericTest.GetMatches()[0] return match.Match } + +// NewGenericTestAndValidate creates a generic handler from the json template with the var map and validate the outcome +func NewGenericTestAndValidate(templateFile, schemaPath string, values map[string]interface{}) (*tnf.Tester, []reel.Handler) { + tester, handlers, result, err := generic.NewGenericFromMap(templateFile, handlerJSONSchemaFilePath, values) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(result).ToNot(gomega.BeNil()) + gomega.Expect(result.Valid()).To(gomega.BeTrue()) + gomega.Expect(handlers).ToNot(gomega.BeNil()) + gomega.Expect(tester).ToNot(gomega.BeNil()) + + return tester, handlers +} diff --git a/test-network-function/lifecycle/suite.go b/test-network-function/lifecycle/suite.go index adcdc2b6e..e4f471db9 100644 --- a/test-network-function/lifecycle/suite.go +++ b/test-network-function/lifecycle/suite.go @@ -24,9 +24,9 @@ import ( "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/handlers/scaling" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/pkg/utils" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" @@ -292,13 +292,7 @@ func shutdownTest(podNamespace, podName string) { values["POD_NAMESPACE"] = podNamespace values["POD_NAME"] = podName values["GO_TEMPLATE_PATH"] = relativeShutdownTestDirectoryPath - tester, handlers, result, err := generic.NewGenericFromMap(relativeShutdownTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) + tester, handlers := utils.NewGenericTestAndValidate(relativeShutdownTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) @@ -396,14 +390,7 @@ func uncordonNode(node string) { context := common.GetContext() values := make(map[string]interface{}) values["NODE"] = node - tester, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(tester).ToNot(gomega.BeNil()) - + tester, handlers := utils.NewGenericTestAndValidate(relativeNodesTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) @@ -436,13 +423,7 @@ func podAntiAffinity(deployment, podNamespace string, replica int) { values := make(map[string]interface{}) values["DEPLOYMENT_NAME"] = deployment values["DEPLOYMENT_NAMESPACE"] = podNamespace - tester, handlers, result, err := generic.NewGenericFromMap(relativePodTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(tester).ToNot(gomega.BeNil()) + tester, handlers := utils.NewGenericTestAndValidate(relativePodTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) @@ -497,13 +478,7 @@ func testImagePolicy(env *config.TestEnvironment) { values["POD_NAME"] = podUnderTest.Name for i := 0; i < ContainerCount; i++ { values["CONTAINER_NUM"] = i - tester, handlers, result, err := generic.NewGenericFromMap(relativeimagepullpolicyTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) + tester, handlers := utils.NewGenericTestAndValidate(relativeimagepullpolicyTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) diff --git a/test-network-function/observability/suite.go b/test-network-function/observability/suite.go index 528fc94d1..26813ea16 100644 --- a/test-network-function/observability/suite.go +++ b/test-network-function/observability/suite.go @@ -26,8 +26,8 @@ import ( "github.com/test-network-function/test-network-function/pkg/config" "github.com/test-network-function/test-network-function/pkg/config/configsections" "github.com/test-network-function/test-network-function/pkg/tnf" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" "github.com/test-network-function/test-network-function/pkg/tnf/testcases" + "github.com/test-network-function/test-network-function/pkg/utils" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" "github.com/test-network-function/test-network-function/test-network-function/results" @@ -82,13 +82,7 @@ func loggingTest(c configsections.ContainerIdentifier) { values["POD_NAMESPACE"] = c.Namespace values["POD_NAME"] = c.PodName values["CONTAINER_NAME"] = c.ContainerName - tester, handlers, result, err := generic.NewGenericFromMap(relativeLoggingTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) + tester, handlers := utils.NewGenericTestAndValidate(relativeLoggingTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) @@ -108,13 +102,7 @@ func testCrds() { values["CRD_NAME"] = crdName values["TIMEOUT"] = testCrdsTimeout.Nanoseconds() - tester, handlers, result, err := generic.NewGenericFromMap(relativeCrdTestPath, common.RelativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(tester).ToNot(gomega.BeNil()) - + tester, handlers := utils.NewGenericTestAndValidate(relativeCrdTestPath, common.RelativeSchemaPath, values) test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(test).ToNot(gomega.BeNil()) gomega.Expect(err).To(gomega.BeNil()) diff --git a/test-network-function/operator/suite.go b/test-network-function/operator/suite.go index 754ccb01e..4d26ec264 100644 --- a/test-network-function/operator/suite.go +++ b/test-network-function/operator/suite.go @@ -21,7 +21,7 @@ import ( "path" "strings" - "github.com/test-network-function/test-network-function/pkg/tnf/handlers/generic" + "github.com/test-network-function/test-network-function/pkg/utils" "github.com/test-network-function/test-network-function/test-network-function/common" "github.com/test-network-function/test-network-function/test-network-function/identifiers" @@ -50,8 +50,8 @@ var ( // pathRelativeToRoot is used to calculate relative filepaths for the `test-network-function` executable entrypoint. pathRelativeToRoot = path.Join("..") - // relativeNodesTestPath is the relative path to the nodes.json test case. - relativeNodesTestPath = path.Join(pathRelativeToRoot, checkSubscriptionTestPath) + // relativecheckSubscriptionTestPath is the relative path to the nodes.json test case. + relativecheckSubscriptionTestPath = path.Join(pathRelativeToRoot, checkSubscriptionTestPath) // relativeSchemaPath is the relative path to the generic-test.schema.json JSON schema. relativeSchemaPath = path.Join(pathRelativeToRoot, schemaPath) @@ -95,13 +95,7 @@ func testOperatorIsInstalledViaOLM(subscriptionName, subscriptionNamespace strin values := make(map[string]interface{}) values["SUBSCRIPTION_NAME"] = subscriptionName values["SUBSCRIPTION_NAMESPACE"] = subscriptionNamespace - tester, handlers, result, err := generic.NewGenericFromMap(relativeNodesTestPath, relativeSchemaPath, values) - gomega.Expect(err).To(gomega.BeNil()) - gomega.Expect(result).ToNot(gomega.BeNil()) - gomega.Expect(result.Valid()).To(gomega.BeTrue()) - gomega.Expect(handlers).ToNot(gomega.BeNil()) - gomega.Expect(len(handlers)).To(gomega.Equal(1)) - gomega.Expect(tester).ToNot(gomega.BeNil()) + tester, handlers := utils.NewGenericTestAndValidate(relativecheckSubscriptionTestPath, relativeSchemaPath, values) context := common.GetContext() test, err := tnf.NewTest(context.GetExpecter(), *tester, handlers, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) From 9e78c1db7baf5a85be1dcaa57003e661710e4c5c Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Mon, 29 Nov 2021 13:07:31 -0600 Subject: [PATCH 176/344] More typo cleanup (#495) --- pkg/tnf/handlers/clusterrolebinding/clusterrolebinding.go | 2 +- pkg/tnf/handlers/clusterversion/clusterversion.go | 2 +- pkg/tnf/handlers/deployments/deployments.go | 2 +- pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go | 2 +- pkg/tnf/handlers/deploymentsnodes/deploymentsnodes.go | 2 +- pkg/tnf/handlers/hugepages/hugepages.go | 2 +- pkg/tnf/handlers/nodenames/nodenames.go | 2 +- pkg/tnf/handlers/nodetainted/nodetainted.go | 2 +- pkg/tnf/handlers/owners/owners.go | 2 +- pkg/tnf/handlers/rolebinding/rolebinding.go | 2 +- pkg/tnf/handlers/scaling/scaling.go | 2 +- pkg/tnf/handlers/scaling/scaling_hpa.go | 2 +- pkg/tnf/handlers/serviceaccount/serviceaccount.go | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/tnf/handlers/clusterrolebinding/clusterrolebinding.go b/pkg/tnf/handlers/clusterrolebinding/clusterrolebinding.go index 10632c9b2..2eb691a60 100644 --- a/pkg/tnf/handlers/clusterrolebinding/clusterrolebinding.go +++ b/pkg/tnf/handlers/clusterrolebinding/clusterrolebinding.go @@ -60,7 +60,7 @@ func (crb *ClusterRoleBinding) Args() []string { return crb.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (crb *ClusterRoleBinding) GetIdentifier() identifier.Identifier { return identifier.ClusterRoleBindingIdentifier } diff --git a/pkg/tnf/handlers/clusterversion/clusterversion.go b/pkg/tnf/handlers/clusterversion/clusterversion.go index c5975ad3b..276436c11 100644 --- a/pkg/tnf/handlers/clusterversion/clusterversion.go +++ b/pkg/tnf/handlers/clusterversion/clusterversion.go @@ -65,7 +65,7 @@ func (ver *TestMetadata) GetVersions() ClusterVersion { return ver.versions } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (ver *TestMetadata) GetIdentifier() identifier.Identifier { return identifier.ClusterVersionIdentifier } diff --git a/pkg/tnf/handlers/deployments/deployments.go b/pkg/tnf/handlers/deployments/deployments.go index 74cd4c72a..615045131 100644 --- a/pkg/tnf/handlers/deployments/deployments.go +++ b/pkg/tnf/handlers/deployments/deployments.go @@ -79,7 +79,7 @@ func (dp *Deployments) Args() []string { return dp.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (dp *Deployments) GetIdentifier() identifier.Identifier { return identifier.DeploymentsIdentifier } diff --git a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go index e8e500ef6..1ee91e8c2 100644 --- a/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go +++ b/pkg/tnf/handlers/deploymentsdrain/deploymentsdrain.go @@ -59,7 +59,7 @@ func (dd *DeploymentsDrain) Args() []string { return dd.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (dd *DeploymentsDrain) GetIdentifier() identifier.Identifier { return identifier.DeploymentsDrainIdentifier } diff --git a/pkg/tnf/handlers/deploymentsnodes/deploymentsnodes.go b/pkg/tnf/handlers/deploymentsnodes/deploymentsnodes.go index 55d4c3470..1a302f45d 100644 --- a/pkg/tnf/handlers/deploymentsnodes/deploymentsnodes.go +++ b/pkg/tnf/handlers/deploymentsnodes/deploymentsnodes.go @@ -67,7 +67,7 @@ func (dn *DeploymentsNodes) Args() []string { return dn.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (dn *DeploymentsNodes) GetIdentifier() identifier.Identifier { return identifier.DeploymentsNodesIdentifier } diff --git a/pkg/tnf/handlers/hugepages/hugepages.go b/pkg/tnf/handlers/hugepages/hugepages.go index 7cce08bac..8b842cb72 100644 --- a/pkg/tnf/handlers/hugepages/hugepages.go +++ b/pkg/tnf/handlers/hugepages/hugepages.go @@ -65,7 +65,7 @@ func (hp *Hugepages) Args() []string { return hp.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (hp *Hugepages) GetIdentifier() identifier.Identifier { return identifier.HugepagesIdentifier } diff --git a/pkg/tnf/handlers/nodenames/nodenames.go b/pkg/tnf/handlers/nodenames/nodenames.go index 16ff155f1..64ba4bfeb 100644 --- a/pkg/tnf/handlers/nodenames/nodenames.go +++ b/pkg/tnf/handlers/nodenames/nodenames.go @@ -71,7 +71,7 @@ func (nn *NodeNames) Args() []string { return nn.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (nn *NodeNames) GetIdentifier() identifier.Identifier { return identifier.NodeNamesIdentifier } diff --git a/pkg/tnf/handlers/nodetainted/nodetainted.go b/pkg/tnf/handlers/nodetainted/nodetainted.go index 320e17897..cd4b03c07 100644 --- a/pkg/tnf/handlers/nodetainted/nodetainted.go +++ b/pkg/tnf/handlers/nodetainted/nodetainted.go @@ -52,7 +52,7 @@ func (nt *NodeTainted) Args() []string { return nt.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (nt *NodeTainted) GetIdentifier() identifier.Identifier { return identifier.NodeTaintedIdentifier } diff --git a/pkg/tnf/handlers/owners/owners.go b/pkg/tnf/handlers/owners/owners.go index e1508d6cf..805db4ded 100644 --- a/pkg/tnf/handlers/owners/owners.go +++ b/pkg/tnf/handlers/owners/owners.go @@ -57,7 +57,7 @@ func (ow *Owners) Args() []string { return ow.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (ow *Owners) GetIdentifier() identifier.Identifier { return identifier.OwnersIdentifier } diff --git a/pkg/tnf/handlers/rolebinding/rolebinding.go b/pkg/tnf/handlers/rolebinding/rolebinding.go index 9719f7b4c..c7d615807 100644 --- a/pkg/tnf/handlers/rolebinding/rolebinding.go +++ b/pkg/tnf/handlers/rolebinding/rolebinding.go @@ -62,7 +62,7 @@ func (rb *RoleBinding) Args() []string { return rb.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (rb *RoleBinding) GetIdentifier() identifier.Identifier { return identifier.RoleBindingIdentifier } diff --git a/pkg/tnf/handlers/scaling/scaling.go b/pkg/tnf/handlers/scaling/scaling.go index d7d9efa34..8997c8438 100644 --- a/pkg/tnf/handlers/scaling/scaling.go +++ b/pkg/tnf/handlers/scaling/scaling.go @@ -55,7 +55,7 @@ func (scaling *Scaling) Args() []string { return scaling.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (scaling *Scaling) GetIdentifier() identifier.Identifier { return identifier.ScalingIdentifier } diff --git a/pkg/tnf/handlers/scaling/scaling_hpa.go b/pkg/tnf/handlers/scaling/scaling_hpa.go index ed4836f2d..e72f171fa 100644 --- a/pkg/tnf/handlers/scaling/scaling_hpa.go +++ b/pkg/tnf/handlers/scaling/scaling_hpa.go @@ -55,7 +55,7 @@ func (hpascaling *HpAScaling) Args() []string { return hpascaling.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (hpascaling *HpAScaling) GetIdentifier() identifier.Identifier { return identifier.ScalingIdentifier } diff --git a/pkg/tnf/handlers/serviceaccount/serviceaccount.go b/pkg/tnf/handlers/serviceaccount/serviceaccount.go index 6cb2e5c4e..a5d34c624 100644 --- a/pkg/tnf/handlers/serviceaccount/serviceaccount.go +++ b/pkg/tnf/handlers/serviceaccount/serviceaccount.go @@ -51,7 +51,7 @@ func (sa *ServiceAccount) Args() []string { return sa.args } -// GetIdentifier returns the tnf.Test specific identifiesa. +// GetIdentifier returns the tnf.Test specific identifier. func (sa *ServiceAccount) GetIdentifier() identifier.Identifier { return identifier.ServiceAccountIdentifier } From aba57da360ffe5680b1a9f9b37f5ea9810cc5501 Mon Sep 17 00:00:00 2001 From: Shimrit Peretz <34240686+shimritproj@users.noreply.github.com> Date: Mon, 29 Nov 2021 22:02:33 +0200 Subject: [PATCH 177/344] Fix the capability test in case of multi lines from the command output (#493) * Add SCC capability checks for IPC_LOCK and NET_RAW * fixed make lint * Updated the file privilegedpod.yml * fixed make lint * Resolved comment of Jun * Updated file privilegefpod.go * Negative tests * fixed make lint * Resolved comment of Jun and Gonzalo Co-authored-by: Shimrit peretz Co-authored-by: Jun Chen --- pkg/tnf/handlers/container/pod.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/tnf/handlers/container/pod.go b/pkg/tnf/handlers/container/pod.go index 81d85df13..6e0acb4fe 100644 --- a/pkg/tnf/handlers/container/pod.go +++ b/pkg/tnf/handlers/container/pod.go @@ -100,14 +100,15 @@ func (p *Pod) ReelMatch(_, _, match string) *reel.Step { p.result = tnf.SUCCESS return nil } - replacer := strings.NewReplacer(`[`, ``, "\"", ``, `]`, ``, `, `, `,`) + replacer := strings.NewReplacer(`[`, ``, "\"", ``, `]`, ``, "\n", ``, "\t", ``) + match = replacer.Replace(match) f := func(c rune) bool { return c == ',' } matchSlice := strings.FieldsFunc(match, f) for _, status := range matchSlice { - if contains(p.ExpectStatus, status) { + if contains(p.ExpectStatus, strings.Trim(status, " ")) { if p.Action == testcases.Deny { // Single deny match is failure. return nil } From edd78bcabe3cf9da36d40fd5d510c985fa1dc869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Pe=C3=B1a?= Date: Tue, 30 Nov 2021 15:22:44 +0100 Subject: [PATCH 178/344] Add static HTML page to visualize claim.json (#494) * Add static HTML page to visualize claim.json As part of the CI process, we want to make it easier to check the claim.json file after a test suite execution. This commit adds a results.html file, which can parse a claim file and show its output in a human-readable way. * Fix typo in results.html for failed tests Co-authored-by: Jun Chen --- .github/workflows/pre-main.yaml | 2 + script/results.html | 256 ++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 script/results.html diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 42c815a78..0a2a88035 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -153,6 +153,7 @@ jobs: path: | test-network-function/*.xml test-network-function/claim.json + script/results.html # Perform smoke tests using a TNF container. @@ -189,6 +190,7 @@ jobs: path: | ${{ env.TNF_OUTPUT_DIR }}/*.xml ${{ env.TNF_OUTPUT_DIR }}/claim.json + script/results.html # Push the new unstable TNF image to Quay.io. - name: (if on main and upstream) Authenticate against Quay.io diff --git a/script/results.html b/script/results.html new file mode 100644 index 000000000..a518dd4e5 --- /dev/null +++ b/script/results.html @@ -0,0 +1,256 @@ + + + + + TNF claim.json parser + + + + + +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+ +
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ + + + + + + + + + + + From 4a8ef3fa1d3f12b202798b769e0cd0ec8b26b2d0 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Tue, 30 Nov 2021 19:11:57 +0100 Subject: [PATCH 179/344] Added claim file logger function in tnf (test.go) package (#498) Co-authored-by: Jun Chen --- pkg/tnf/test.go | 12 ++++++++++++ test-network-function/accesscontrol/suite.go | 20 ++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/tnf/test.go b/pkg/tnf/test.go index 8ae9126e1..c80734a99 100644 --- a/pkg/tnf/test.go +++ b/pkg/tnf/test.go @@ -17,14 +17,26 @@ package tnf import ( + "fmt" "time" expect "github.com/google/goexpect" + "github.com/onsi/ginkgo" "github.com/onsi/gomega" + "github.com/sirupsen/logrus" "github.com/test-network-function/test-network-function/pkg/tnf/identifier" "github.com/test-network-function/test-network-function/pkg/tnf/reel" ) +// ClaimFilePrintf prints to claim and junit report files. +func ClaimFilePrintf(format string, args ...interface{}) { + message := fmt.Sprintf(format+"\n", args...) + _, err := ginkgo.GinkgoWriter.Write([]byte(message)) + if err != nil { + logrus.Errorf("Ginkgo writer could not write msg '%s' because: %s", message, err) + } +} + const ( // ERROR represents an errored test. ERROR = iota diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index 8ed8c691d..e6eaae1c0 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -52,14 +52,6 @@ var ( "istio-", "aspenmesh-", } - - tcClaimLogPrintf = func(format string, args ...interface{}) { - message := fmt.Sprintf(format+"\n", args...) - _, err := ginkgo.GinkgoWriter.Write([]byte(message)) - if err != nil { - log.Errorf("Ginkgo writer could not write msg '%s' because: %s", message, err) - } - } ) var _ = ginkgo.Describe(common.AccessControlTestKey, func() { @@ -164,7 +156,7 @@ func getCrsNamespaces(crdName, crdKind string) (map[string]string, error) { gomega.Expect(crdKind).NotTo(gomega.BeEmpty()) getCrNamespaceCommand := fmt.Sprintf(ocGetCrNamespaceFormat, crdKind) cmdOut := utils.ExecuteCommand(getCrNamespaceCommand, common.DefaultTimeout, common.GetContext(), func() { - tcClaimLogPrintf("CRD %s: Failed to get CRs (kind=%s)", crdName, crdKind) + tnf.ClaimFilePrintf("CRD %s: Failed to get CRs (kind=%s)", crdName, crdKind) }) crNamespaces := map[string]string{} @@ -191,7 +183,7 @@ func testCrsNamespaces(crNames, configNamespaces []string) (invalidCrs map[strin for _, crdName := range crNames { getCrPluralNameCommand := fmt.Sprintf(ocGetCrPluralNameFormat, crdName) crdPluralName := utils.ExecuteCommand(getCrPluralNameCommand, common.DefaultTimeout, common.GetContext(), func() { - tcClaimLogPrintf("CRD %s: Failed to get CR plural name.", crdName) + tnf.ClaimFilePrintf("CRD %s: Failed to get CR plural name.", crdName) }) crNamespaces, err := getCrsNamespaces(crdName, crdPluralName) @@ -211,7 +203,7 @@ func testCrsNamespaces(crNames, configNamespaces []string) (invalidCrs map[strin } if !found { - tcClaimLogPrintf("CRD: %s (kind:%s) - CR %s has an invalid namespace (%s)", crdName, crdPluralName, crName, namespace) + tnf.ClaimFilePrintf("CRD: %s (kind:%s) - CR %s has an invalid namespace (%s)", crdName, crdPluralName, crName, namespace) if crNames, exists := invalidCrs[crdName]; exists { invalidCrs[crdName] = append(crNames, crName) } else { @@ -233,7 +225,7 @@ func testNamespace(env *config.TestEnvironment) { ginkgo.By(fmt.Sprintf("Checking namespace %s", namespace)) for _, invalidPrefix := range invalidNamespacePrefixes { if strings.HasPrefix(namespace, invalidPrefix) { - tcClaimLogPrintf("Namespace %s has invalid prefix %s", namespace, invalidPrefix) + tnf.ClaimFilePrintf("Namespace %s has invalid prefix %s", namespace, invalidPrefix) failedNamespaces = append(failedNamespaces, namespace) } } @@ -247,7 +239,7 @@ func testNamespace(env *config.TestEnvironment) { if nonValidPodsNum := len(env.Config.NonValidPods); nonValidPodsNum > 0 { for _, invalidPod := range env.Config.NonValidPods { - tcClaimLogPrintf("Pod %s has invalid namespace %s", invalidPod.Name, invalidPod.Namespace) + tnf.ClaimFilePrintf("Pod %s has invalid namespace %s", invalidPod.Name, invalidPod.Namespace) } ginkgo.Fail(fmt.Sprintf("Found %d pods under test belonging to invalid namespaces.", nonValidPodsNum)) @@ -259,7 +251,7 @@ func testNamespace(env *config.TestEnvironment) { if invalidCrsNum := len(invalidCrs); invalidCrsNum > 0 { for crdName, crs := range invalidCrs { for _, crName := range crs { - tcClaimLogPrintf("CRD %s - CR %s has an invalid namespace.", crdName, crName) + tnf.ClaimFilePrintf("CRD %s - CR %s has an invalid namespace.", crdName, crName) } } ginkgo.Fail(fmt.Sprintf("Found %d CRs belonging to invalid namespaces.", invalidCrsNum)) From b5a8cc10204aa28b566bd41a8d89d66cb52439d4 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 1 Dec 2021 08:48:07 -0600 Subject: [PATCH 180/344] Add unit tests for containerid package (#497) --- .../handlers/containerid/containerid_test.go | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 pkg/tnf/handlers/containerid/containerid_test.go diff --git a/pkg/tnf/handlers/containerid/containerid_test.go b/pkg/tnf/handlers/containerid/containerid_test.go new file mode 100644 index 000000000..82545b1b5 --- /dev/null +++ b/pkg/tnf/handlers/containerid/containerid_test.go @@ -0,0 +1,76 @@ +// Copyright (C) 2021 Red Hat, Inc. +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +package containerid_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/test-network-function/test-network-function/pkg/tnf" + "github.com/test-network-function/test-network-function/pkg/tnf/handlers/containerid" +) + +func TestArgs(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + assert.Equal(t, "cat /proc/self/cgroup", strings.Join(c.Args(), " ")) +} + +func TestGetIdentifier(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + identifier := c.GetIdentifier() + assert.Equal(t, "http://test-network-function.com/tests/generic/containerId", identifier.URL) + assert.Equal(t, "v1.0.0", identifier.SemanticVersion) +} + +func TestTimeout(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + assert.Equal(t, time.Second*5, c.Timeout()) +} + +func TestResult(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + assert.Equal(t, tnf.ERROR, c.Result()) +} + +func TestReelFirst(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + rf := c.ReelFirst() + assert.Equal(t, containerid.SuccessfulOutputRegex, rf.Expect[0]) + assert.Equal(t, time.Second*5, rf.Timeout) +} + +func TestReelMatch(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + step := c.ReelMatch("", "", "crio-test.scope") + assert.Nil(t, step) + assert.Equal(t, tnf.SUCCESS, c.Result()) + step = c.ReelMatch("", "", "crio-test-scope") + assert.Nil(t, step) + assert.Equal(t, tnf.FAILURE, c.Result()) +} + +func TestReelTimeout(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + assert.Nil(t, c.ReelTimeout()) +} + +func TestGetID(t *testing.T) { + c := containerid.NewContainerID(5 * time.Second) + assert.Equal(t, c.GetID(), "") +} From 1e65ec4744548332ab134179693e93f851188cac Mon Sep 17 00:00:00 2001 From: edcdavid <86730676+edcdavid@users.noreply.github.com> Date: Wed, 1 Dec 2021 10:41:07 -0600 Subject: [PATCH 181/344] Fixing Multus skip logic (#499) Co-authored-by: Jun Chen --- test-network-function/networking/suite.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test-network-function/networking/suite.go b/test-network-function/networking/suite.go index d39f97bd4..59a3e632f 100644 --- a/test-network-function/networking/suite.go +++ b/test-network-function/networking/suite.go @@ -82,6 +82,8 @@ func testDefaultNetworkConnectivity(env *config.TestEnvironment, count int) { found := false for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { + tnf.ClaimFilePrintf("Skipping container %s because it is excluded from connectivity tests (default)", cut.ContainerConfiguration.PodName) + continue } found = true @@ -113,12 +115,14 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { found := false for _, cut := range env.ContainersUnderTest { if _, ok := env.ContainersToExcludeFromConnectivityTests[cut.ContainerIdentifier]; ok { + tnf.ClaimFilePrintf("Skipping container %s because it is excluded from connectivity tests (multus)", cut.ContainerConfiguration.PodName) continue } - found = true if len(cut.ContainerConfiguration.MultusIPAddresses) == 0 { - ginkgo.Skip("No Multus IPs detected") + tnf.ClaimFilePrintf("Skipping container %s for multus test because no multus IPs are present", cut.ContainerConfiguration.PodName) + continue } + found = true for _, multusIPAddress := range cut.ContainerConfiguration.MultusIPAddresses { testOrchestrator := env.TestOrchestrator @@ -127,9 +131,9 @@ func testMultusNetworkConnectivity(env *config.TestEnvironment, count int) { multusIPAddress)) testPing(testOrchestrator.Oc, multusIPAddress, count) } - if !found { - ginkgo.Skip("No container found suitable for Multus connectivity test") - } + } + if !found { + ginkgo.Skip("No container found suitable for Multus connectivity test") } }) }) From 2f4dfaac0e9bea4810e165fbfcc21e12643f65dd Mon Sep 17 00:00:00 2001 From: Jun Chen Date: Wed, 1 Dec 2021 13:49:10 -0500 Subject: [PATCH 182/344] Fix for ReelMatch() always getting called (#496) * Fix for reel.Match always getting called * Fixes for impacted tests and import update * Fix lint and unit test * Enhanced unit test to match the new reel step implementation Co-authored-by: edcdavid <86730676+edcdavid@users.noreply.github.com> --- go.mod | 2 + go.sum | 4 +- pkg/tnf/handlers/command/command.json | 2 +- pkg/tnf/handlers/container/pod.go | 2 +- pkg/tnf/handlers/container/pod_test.go | 2 +- pkg/tnf/reel/reel.go | 40 +++++------ pkg/tnf/test_test.go | 80 +++++++++++++++++---- pkg/tnf/testcases/data/cnf/privilegedpod.go | 5 +- 8 files changed, 96 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 6daf619a3..d8433795d 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/test-network-function/test-network-function go 1.17 +replace github.com/google/goexpect => github.com/test-network-function/goexpect v0.0.1 + require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/basgys/goxml2json v1.1.0 diff --git a/go.sum b/go.sum index 46fc2bee5..fd74ad129 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97 h1:/nu0LtOLsZMlqfk6i50C5fmo5cp5WfciWggZYlwGNAg= -github.com/google/goexpect v0.0.0-20210330220015-096e5d1cbd97/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4= github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4= @@ -262,6 +260,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/test-network-function/goexpect v0.0.1 h1:eX+QxxHNXKPQubIZZ1qI/GsN/owXCqtoRCS72JdBQVs= +github.com/test-network-function/goexpect v0.0.1/go.mod h1:hpGtPP1qpC3lzxn0akwYH3/TdDl/XW6SSmyMivkdleI= github.com/test-network-function/test-network-function-claim v1.0.5 h1:6MSI0hX/6Z5JTHIxg2n+IQF8qBHAKxvZRdwpdMW8eC8= github.com/test-network-function/test-network-function-claim v1.0.5/go.mod h1:jPVWu2/YQ0JbY3LHcL+BuZ48VjBLeUaNzh3QqWip4CE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= diff --git a/pkg/tnf/handlers/command/command.json b/pkg/tnf/handlers/command/command.json index 400468de0..75ca56c32 100644 --- a/pkg/tnf/handlers/command/command.json +++ b/pkg/tnf/handlers/command/command.json @@ -9,7 +9,7 @@ "testTimeout": {{ .TIMEOUT }}, "reelFirstStep": { "execute": "{{ .COMMAND }}", - "expect":[ "(?m).*"], + "expect": ["(?m).*"], "timeout": {{ .TIMEOUT }} }, "resultContexts":[ diff --git a/pkg/tnf/handlers/container/pod.go b/pkg/tnf/handlers/container/pod.go index 6e0acb4fe..88af8b16b 100644 --- a/pkg/tnf/handlers/container/pod.go +++ b/pkg/tnf/handlers/container/pod.go @@ -71,7 +71,7 @@ func (p *Pod) Result() int { // ReelFirst returns a step which expects an pod status for the given pod. func (p *Pod) ReelFirst() *reel.Step { return &reel.Step{ - Expect: []string{testcases.GetOutRegExp(testcases.AllowAll)}, + Expect: []string{testcases.GetOutRegExp(testcases.AllowEmpty)}, Timeout: p.timeout, } } diff --git a/pkg/tnf/handlers/container/pod_test.go b/pkg/tnf/handlers/container/pod_test.go index 40b3faf1d..62913e089 100644 --- a/pkg/tnf/handlers/container/pod_test.go +++ b/pkg/tnf/handlers/container/pod_test.go @@ -60,7 +60,7 @@ func TestPod_ReelFirst(t *testing.T) { c := container.NewPod(args, name, namespace, stringExpectedStatus, testcases.StringType, testcases.Allow, testTimeoutDuration) step := c.ReelFirst() assert.Equal(t, "", step.Execute) - assert.Equal(t, []string{testcases.GetOutRegExp(testcases.AllowAll)}, step.Expect) + assert.Equal(t, []string{testcases.GetOutRegExp(testcases.AllowEmpty)}, step.Expect) assert.Equal(t, testTimeoutDuration, step.Timeout) } diff --git a/pkg/tnf/reel/reel.go b/pkg/tnf/reel/reel.go index 8ce247639..0d4855cda 100644 --- a/pkg/tnf/reel/reel.go +++ b/pkg/tnf/reel/reel.go @@ -187,13 +187,12 @@ func (r *Reel) Step(step *Step, handler Handler) error { return r.Err } exec, exp, timeout := step.unpack() - var batcher []expect.Batcher - batcher = r.generateBatcher(exec) + var batchers []expect.Batcher + batchers = r.generateBatcher(exec) // firstMatchRe is the first regular expression (expectation) that has matched results var firstMatchRe string - batcher = r.batchExpectations(exp, batcher, &firstMatchRe) - results, err := (*r.expecter).ExpectBatch(batcher, timeout) - + batchers = r.batchExpectations(exp, batchers, &firstMatchRe) + results, err := (*r.expecter).ExpectBatch(batchers, timeout) if !step.hasExpectations() { return nil } @@ -206,32 +205,31 @@ func (r *Reel) Step(step *Step, handler Handler) error { } else { if len(results) > 0 { result := results[0] - output, outputStatus := r.stripEmulatedPromptFromOutput(result.Output) if outputStatus != 0 { return fmt.Errorf("error executing command exit code:%d", outputStatus) } match, matchStatus := r.stripEmulatedPromptFromOutput(result.Match[0]) - log.Debugf("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d", output, match, outputStatus, matchStatus) - - matchIndex := strings.Index(output, match) - var before string - // special case: the match regex may be nothing at all. - if matchIndex > 0 { - before = output[0 : matchIndex-1] + log.Debugf("command status: output=%s, match=%s, outputStatus=%d, matchStatus=%d, caseIndex=%d", output, match, outputStatus, matchStatus, result.CaseIdx) + // Check if the matching case is the extra one added in generateCases() for prompt return in error cases, skip calling ReelMatch if it is + if result.CaseIdx != len(batchers[result.Idx].Cases())-1 { + matchIndex := strings.Index(output, match) + var before string + // special case: the match regex may be nothing at all. + if matchIndex > 0 { + before = output[0 : matchIndex-1] + } else { + before = "" + } + strippedFirstMatchRe := r.stripEmulatedRegularExpression(firstMatchRe) + step = handler.ReelMatch(strippedFirstMatchRe, before, match) } else { - before = "" + step = nil } - strippedFirstMatchRe := r.stripEmulatedRegularExpression(firstMatchRe) - step = handler.ReelMatch(strippedFirstMatchRe, before, match) } } - // This is for the last step - if r.Err != nil { - return r.Err - } } - return nil + return r.Err } // Run the target subprocess to completion. The first step to take is supplied by handler. Consequent steps are diff --git a/pkg/tnf/test_test.go b/pkg/tnf/test_test.go index 8fa5e3a45..6a5f441c1 100644 --- a/pkg/tnf/test_test.go +++ b/pkg/tnf/test_test.go @@ -102,10 +102,27 @@ type testRunTestCase struct { reelMatchBefore string reelMatchMatch string reelMatchResult *reel.Step + testRunErr error } -func fakeSentinelOutput() string { - return fmt.Sprintf("someOutput%s\n", reel.EndOfTestRegexPostfix) +func fakeSentinelOutputWithReturnCode(output string, code int) string { + return fmt.Sprintf("%s%s %s%d\n", output, reel.EndOfTestSentinel, reel.ExitKeyword, code) +} + +func fakeSentinelOutput(output string) string { + return fakeSentinelOutputWithReturnCode(output, 0) +} + +func fakeOutput() string { + return "someOutput" +} + +func fakeWrongOutput() string { + return "something else" +} + +func fakeErrorCode() int { + return 1 } // Tests the actual state machine. @@ -136,24 +153,64 @@ var testRunTestCases = map[string]testRunTestCase{ testCommandArgs: defaultTestCommand, reelFirstResult: &reel.Step{ Execute: "ls", - Expect: []string{fakeSentinelOutput()}, + Expect: []string{fakeOutput()}, Timeout: testTimeoutDuration, }, testerResultResult: tnf.ERROR, expectBatchIsCalled: true, expectBatchBatchResResult: []expect.BatchRes{ { - Idx: 0, - Output: fakeSentinelOutput(), - Match: []string{fakeSentinelOutput()}}, + Idx: 1, + CaseIdx: 0, + Output: fakeSentinelOutput(fakeOutput()), + Match: []string{fakeSentinelOutput(fakeOutput())}}, }, expectBatchBatchResErr: nil, reelMatchIsCalled: true, reelMatchPattern: "", reelMatchBefore: "", - reelMatchMatch: fakeSentinelOutput(), + reelMatchMatch: fakeOutput(), reelMatchResult: nil, }, + "reel_first_only": { + testCommandArgs: defaultTestCommand, + reelFirstResult: &reel.Step{ + Execute: "ls", + Expect: []string{fakeOutput()}, + Timeout: testTimeoutDuration, + }, + testerResultResult: tnf.ERROR, + expectBatchIsCalled: true, + expectBatchBatchResResult: []expect.BatchRes{ + { + Idx: 1, + CaseIdx: 1, + Output: fakeSentinelOutput(fakeWrongOutput()), + Match: []string{fakeSentinelOutput(fakeWrongOutput())}}, + }, + expectBatchBatchResErr: nil, + reelMatchIsCalled: false, + }, + "reel_first_only_with_error_code": { + testCommandArgs: defaultTestCommand, + reelFirstResult: &reel.Step{ + Execute: "ls", + Expect: []string{fakeOutput()}, + Timeout: testTimeoutDuration, + }, + testerResultResult: tnf.ERROR, + expectBatchIsCalled: true, + expectBatchBatchResResult: []expect.BatchRes{ + { + Idx: 1, + CaseIdx: 1, + Output: fakeSentinelOutputWithReturnCode(fakeWrongOutput(), fakeErrorCode()), + Match: []string{fakeSentinelOutputWithReturnCode(fakeWrongOutput(), fakeErrorCode())}}, + }, + expectBatchBatchResErr: nil, + reelMatchIsCalled: false, + testRunErr: fmt.Errorf("error executing command exit code:%d", fakeErrorCode()), + }, } // Also covers ReelFirst() and ReelMatch(). Tests those state transitions. @@ -163,7 +220,7 @@ func TestTest_Run(t *testing.T) { for _, testCase := range testRunTestCases { mockExpecter := mock_interactive.NewMockExpecter(ctrl) - testCommand := strings.Join(testCase.testCommandArgs, " ") + "\n" + testCommand := fmt.Sprintf("%s ; echo %s %s$?\n", strings.Join(testCase.testCommandArgs, " "), reel.EndOfTestSentinel, reel.ExitKeyword) mockExpecter.EXPECT().Send(testCommand).AnyTimes() // Only for test cases where ReelMatch(...) is encountered. @@ -183,13 +240,12 @@ func TestTest_Run(t *testing.T) { var expecter expect.Expecter = mockExpecter var errorChannel <-chan error - test, err := tnf.NewTest(&expecter, mockTester, []reel.Handler{mockHandler}, errorChannel, reel.DisableTerminalPromptEmulation()) + test, err := tnf.NewTest(&expecter, mockTester, []reel.Handler{mockHandler}, errorChannel) assert.Nil(t, err) assert.NotNil(t, test) result, err := test.Run() - // Since we have no control over the t.runner, just make the assertion that err is nil. In these cases, it - // always should be nil, as it is mocked. - assert.Nil(t, err) + + assert.Equal(t, err, testCase.testRunErr) assert.Equal(t, result, testCase.testerResultResult) } } diff --git a/pkg/tnf/testcases/data/cnf/privilegedpod.go b/pkg/tnf/testcases/data/cnf/privilegedpod.go index 57baf5e6d..fa04251b2 100644 --- a/pkg/tnf/testcases/data/cnf/privilegedpod.go +++ b/pkg/tnf/testcases/data/cnf/privilegedpod.go @@ -17,7 +17,6 @@ package cnf // PrivilegedPodJSON test templates for privileged pods -//nolint:lll var PrivilegedPodJSON = string(`{ "testcase": [ { @@ -34,7 +33,7 @@ var PrivilegedPodJSON = string(`{ "name": "HOST_PORT_CHECK", "skiptest": true, "loop": 1, - "command": "oc get pod %s -n %s -o go-template='{{$putName := .metadata.name}}{{$cut := (index .spec.containers %d)}}{{range $cut.ports }}{{if .hostPort}}PUT {{$putName}} - CUT {{$cut.name}} has declared hostPort {{.hostPort}}{{\"\\n\"}}{{end}}{{end}}'", + "command": "oc get pod %s -n %s -o go-template='{{range (index .spec.containers %d).ports }}{{.hostPort}}{{end}}'", "action": "allow", "expectedstatus": [ "^()*$" @@ -44,7 +43,7 @@ var PrivilegedPodJSON = string(`{ "name": "HOST_PATH_CHECK", "skiptest": true, "loop": 0, - "command": "oc get pods %s -n %s -o go-template='{{ range .spec.volumes}}{{.hostPath.path}}{{end}}'", + "command": "oc get pods %s -n %s -o go-template='{{range .spec.volumes}}{{.hostPath.path}}{{end}}'", "action": "allow", "expectedstatus": [ "^()*$" From e11a65f3eda34926abfa210aa7621406a2ca1079 Mon Sep 17 00:00:00 2001 From: Gonzalo Reyero Ferreras <87083379+greyerof@users.noreply.github.com> Date: Thu, 2 Dec 2021 17:21:24 +0100 Subject: [PATCH 183/344] Combo TC improvements. (#501) * Combo TC improvements. + Avoid combo TC to stop when one pod fails the TC. + Improved error logging, including the command that was sent and the expectations for the failing TC. * Addressing David's comment Co-authored-by: Jun Chen --- test-network-function/accesscontrol/suite.go | 47 +++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/test-network-function/accesscontrol/suite.go b/test-network-function/accesscontrol/suite.go index e6eaae1c0..1856b3827 100644 --- a/test-network-function/accesscontrol/suite.go +++ b/test-network-function/accesscontrol/suite.go @@ -93,15 +93,31 @@ var _ = ginkgo.Describe(common.AccessControlTestKey, func() { } }) +type failedTcInfo struct { + tc string + containerIdx int + ns string +} + +func addFailedTcInfo(failedTcs map[string][]failedTcInfo, tc, pod, ns string, containerIdx int) { + if tcs, exists := failedTcs[pod]; exists { + tcs = append(tcs, failedTcInfo{tc: tc, containerIdx: containerIdx, ns: ns}) + failedTcs[pod] = tcs + } else { + failedTcs[pod] = []failedTcInfo{{tc: tc, containerIdx: containerIdx, ns: ns}} + } +} + //nolint:gocritic,funlen // ignore hugeParam error. Pointers to loop iterator vars are bad and `testCmd` is likely to be such. func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, testType string) { + const noContainerIdx = -1 testID := identifiers.XformToGinkgoItIdentifierExtended(identifiers.TestHostResourceIdentifier, testCmd.Name) ginkgo.It(testID, func() { context := common.GetContext() + failedTcs := map[string][]failedTcInfo{} // maps a pod name to a slice of failed TCs for _, podUnderTest := range env.PodsUnderTest { podName := podUnderTest.Name podNamespace := podUnderTest.Namespace - ginkgo.By(fmt.Sprintf("Reading namespace of podnamespace= %s podname= %s", podNamespace, podName)) if testCmd.ExpectedType == testcases.Function { for _, val := range testCmd.ExpectedStatus { testCmd.ExpectedStatusFn(podName, testcases.StatusFunctionType(val)) @@ -125,26 +141,47 @@ func runTestOnPods(env *config.TestEnvironment, testCmd testcases.BaseTestCase, if count > 0 { count := 0 for count < podUnderTest.ContainerCount { + ginkgo.By(fmt.Sprintf("Executing TC %s on pod %s (ns %s), container index %d", testCmd.Name, podNamespace, podName, count)) argsCount := append(args, count) - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, argsCount...), " ") + cmd := fmt.Sprintf(testCmd.Command, argsCount...) + cmdArgs := strings.Split(cmd, " ") cnfInTest := containerpkg.NewPod(cmdArgs, podUnderTest.Name, podUnderTest.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) gomega.Expect(cnfInTest).ToNot(gomega.BeNil()) test, err := tnf.NewTest(context.GetExpecter(), cnfInTest, []reel.Handler{cnfInTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) - test.RunAndValidate() + test.RunWithCallbacks(nil, func() { + tnf.ClaimFilePrintf("FAILURE: Command sent: %s, Expectations: %v", cmd, testCmd.ExpectedStatus) + addFailedTcInfo(failedTcs, testCmd.Name, podName, podNamespace, count) + }, func(e error) { + tnf.ClaimFilePrintf("ERROR: Command sent: %s, Expectations: %v, Error: %v", cmd, testCmd.ExpectedStatus, e) + addFailedTcInfo(failedTcs, testCmd.Name, podName, podNamespace, count) + }) count++ } } else { - cmdArgs := strings.Split(fmt.Sprintf(testCmd.Command, args...), " ") + ginkgo.By(fmt.Sprintf("Executing TC %s on pod %s (ns %s)", testCmd.Name, podNamespace, podName)) + cmd := fmt.Sprintf(testCmd.Command, args...) + cmdArgs := strings.Split(cmd, " ") podTest := containerpkg.NewPod(cmdArgs, podUnderTest.Name, podUnderTest.Namespace, testCmd.ExpectedStatus, testCmd.ResultType, testCmd.Action, common.DefaultTimeout) gomega.Expect(podTest).ToNot(gomega.BeNil()) test, err := tnf.NewTest(context.GetExpecter(), podTest, []reel.Handler{podTest}, context.GetErrorChannel()) gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(test).ToNot(gomega.BeNil()) - test.RunAndValidate() + test.RunWithCallbacks(nil, func() { + tnf.ClaimFilePrintf("FAILURE: Command sent: %s, Expectations: %v", cmd, testCmd.ExpectedStatus) + addFailedTcInfo(failedTcs, testCmd.Name, podName, podNamespace, noContainerIdx) + }, func(e error) { + tnf.ClaimFilePrintf("ERROR: Command sent: %s, Expectations: %v, Error: %v", cmd, testCmd.ExpectedStatus, e) + addFailedTcInfo(failedTcs, testCmd.Name, podName, podNamespace, noContainerIdx) + }) } } + + if n := len(failedTcs); n > 0 { + log.Debugf("Failed TCs: %+v", failedTcs) + ginkgo.Fail(fmt.Sprintf("%d pods failed the test.", n)) + } }) } From 2db4a356b547ec11948bead6928a14b229b140d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Pe=C3=B1a?= Date: Thu, 2 Dec 2021 17:47:16 +0100 Subject: [PATCH 184/344] Insert claim.json file in results HTML page (#500) To simplify log viewing, we want to have the CI job insert the claim.json file automatically inside the results.html page. This cannot be done by dynamically loading it due to CORS restrictions, but we can convert the JSON file into a javascript file, then load it from the HTML page. Co-authored-by: Jun Chen --- .github/workflows/pre-main.yaml | 6 ++++-- Dockerfile | 2 ++ run-cnf-suites.sh | 12 ++++++++++++ script/results.html | 11 ++++++++++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 0a2a88035..ea0ea90cc 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -153,7 +153,8 @@ jobs: path: | test-network-function/*.xml test-network-function/claim.json - script/results.html + test-network-function/claimjson.js + test-network-function/results.html # Perform smoke tests using a TNF container. @@ -190,7 +191,8 @@ jobs: path: | ${{ env.TNF_OUTPUT_DIR }}/*.xml ${{ env.TNF_OUTPUT_DIR }}/claim.json - script/results.html + ${{ env.TNF_OUTPUT_DIR }}/claimjson.js + ${{ env.TNF_OUTPUT_DIR }}/results.html # Push the new unstable TNF image to Quay.io. - name: (if on main and upstream) Authenticate against Quay.io diff --git a/Dockerfile b/Dockerfile index eb72f82f5..1031728bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,6 +74,8 @@ RUN make install-tools && \ # Extract what's needed to run at a seperate location RUN mkdir ${TNF_BIN_DIR} && \ cp run-cnf-suites.sh ${TNF_DIR} && \ + mkdir ${TNF_DIR}/script && \ + cp script/results.html ${TNF_DIR}/script && \ # copy all JSON files to allow tests to run cp --parents `find -name \*.json*` ${TNF_DIR} && \ # copy all go template files to allow tests to run diff --git a/run-cnf-suites.sh b/run-cnf-suites.sh index ed54ce688..e0d3a91d9 100755 --- a/run-cnf-suites.sh +++ b/run-cnf-suites.sh @@ -20,6 +20,7 @@ usage_error() { FOCUS="" SKIP="" +BASEDIR=$(dirname $(realpath $0)) # Parge args beginning with "-" while [[ $1 == -* ]]; do case "$1" in @@ -47,6 +48,17 @@ done # specify Junit report file name. GINKGO_ARGS="-junit $OUTPUT_LOC -claimloc $OUTPUT_LOC --ginkgo.junit-report $OUTPUT_LOC/cnf-certification-tests_junit.xml -ginkgo.v -test.v" +# Make sure the HTML output is copied to the output directory, +# even in case of a test failure +function html_output() { + if [ -f ${OUTPUT_LOC}/claim.json ]; then + echo -n "var initialjson=" > ${OUTPUT_LOC}/claimjson.js + cat ${OUTPUT_LOC}/claim.json >> ${OUTPUT_LOC}/claimjson.js + fi + cp ${BASEDIR}/script/results.html ${OUTPUT_LOC} +} +trap html_output EXIT + # If no focus is set then display usage and quit with a non-zero exit code. [ -z "$FOCUS" ] && echo "no focus found" && usage_error diff --git a/script/results.html b/script/results.html index a518dd4e5..58ccfc37d 100644 --- a/script/results.html +++ b/script/results.html @@ -229,8 +229,17 @@ $(text).appendTo($(element)) } - + +