diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5b80c7ca556e22848dfd873ffd05ebdb65c0fbb..2284226676953eccfd8a42db29894a0b57cdc466 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: - post-test - review-prepare - review + - sec-dast - qa - post-qa - pages @@ -61,6 +62,7 @@ variables: ES_JAVA_OPTS: "-Xms256m -Xmx256m" ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200" DOCKER_VERSION: "19.03.0" + DAST_TEST: "true" include: - local: .gitlab/ci/build-images.gitlab-ci.yml diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 9a81ea513b784074f963a6b88e4386869f31fbd9..872343867aaff5430d9e2083fe6b11dcf22f469d 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -3,7 +3,6 @@ - .default-retry - .qa-cache stage: test - needs: [] before_script: - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' - cd qa/ diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 4cef4ee26ff8c4d6adc524ad4ca999329aa3fd7c..d1d00bc77595f4fb4066b33b567e5f667708e9c1 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -207,24 +207,36 @@ downtime_check: - bundle exec rake downtime_check rspec migration pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11 - .rspec-base-migration - .rspec-migration-parallel rspec unit pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11 - .rails:rules:ee-and-foss-unit - .rspec-unit-parallel rspec integration pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11 - .rails:rules:ee-and-foss-integration - .rspec-integration-parallel rspec system pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11 - .rails:rules:ee-and-foss-system @@ -340,6 +352,9 @@ rspec:coverage: ################################################## # EE: default refs (MRs, master, schedules) jobs # rspec migration pg11-as-if-foss: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11-as-if-foss - .rspec-base-migration @@ -347,24 +362,36 @@ rspec migration pg11-as-if-foss: - .rspec-migration-parallel rspec unit pg11-as-if-foss: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11-as-if-foss - .rails:rules:as-if-foss-unit - .rspec-unit-parallel rspec integration pg11-as-if-foss: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11-as-if-foss - .rails:rules:as-if-foss-integration - .rspec-integration-parallel rspec system pg11-as-if-foss: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11-as-if-foss - .rails:rules:as-if-foss-system - .rspec-system-parallel rspec-ee migration pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg11 - .rspec-base-migration @@ -372,35 +399,53 @@ rspec-ee migration pg11: - .rspec-ee-migration-parallel rspec-ee unit pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg11 - .rails:rules:ee-only-unit - .rspec-ee-unit-parallel rspec-ee integration pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg11 - .rails:rules:ee-only-integration - .rspec-ee-integration-parallel rspec-ee system pg11: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg11 - .rails:rules:ee-only-system - .rspec-ee-system-parallel rspec-ee unit pg11 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg11 - .rails:rules:ee-only-unit - .rspec-ee-unit-geo-parallel rspec-ee integration pg11 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg11 - .rails:rules:ee-only-integration rspec-ee system pg11 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg11 - .rails:rules:ee-only-system @@ -418,6 +463,9 @@ db:rollback geo: ########################################## # EE/FOSS: master nightly scheduled jobs # rspec migration pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg12 - .rspec-base-migration @@ -425,18 +473,27 @@ rspec migration pg12: - .rspec-migration-parallel rspec unit pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage - .rspec-unit-parallel rspec integration pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage - .rspec-integration-parallel rspec system pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage @@ -447,6 +504,9 @@ rspec system pg12: ##################################### # EE: master nightly scheduled jobs # rspec-ee migration pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg12 - .rspec-base-migration @@ -454,35 +514,53 @@ rspec-ee migration pg12: - .rspec-ee-migration-parallel rspec-ee unit pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-parallel rspec-ee integration pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only - .rspec-ee-integration-parallel rspec-ee system pg12: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel rspec-ee unit pg12 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-geo-parallel rspec-ee integration pg12 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only rspec-ee system pg12 geo: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-ee-base-geo-pg12 - .rails:rules:master-schedule-nightly--code-backstage-ee-only @@ -492,6 +570,9 @@ rspec-ee system pg12 geo: ################################################## # EE: Canonical MR pipelines rspec foss-impact: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .rspec-base-pg11-as-if-foss - .rails:rules:ee-mr-only diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index a544ac410dd9e20968752a88af281101552d9804..52e5f089f4b215e37adece8bd34c84884f1040a5 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -187,3 +187,152 @@ dependency_scanning: # - .reports:schedule-dast # variables: # DAST_FULL_SCAN_ENABLED: "true" +# DAST-prepare: +# stage: review-after +# image: alpine +# variables: +# TRACE: 1 +# needs: ["review-deploy"] +# script: +# - apk add jq curl +# - REVIEW_APPS_ROOT_TOKEN=$(cat environment_token.txt) +# - CI_ENVIRONMENT_URL=$(cat environment_url.txt) +# - source scripts/review_apps/seed-dast-test-data.sh +# - USERID=$(create_user "user1") +# - create_project_for_user $USERID +# - USERID=$(create_user "user2") +# - create_project_for_user $USERID +# - USERID=$(create_user "user3") +# - create_project_for_user $USERID +# - USERID=$(create_user "user4") +# - create_project_for_user $USERID +# - USERID=$(create_user "user5") +# - create_project_for_user $USERID +.dast_conf: + tags: + - prm + extends: + - .reports:rules:dast + - .default-retry + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + variables: + # DAST_USERNAME: "root" + DAST_USERNAME: "root" + DAST_USERNAME_FIELD: "user[login]" + DAST_PASSWORD_FIELD: "user[password]" + DAST_FULL_SCAN_ENABLED: "true" + DAST_SPIDER_MINS: 0 + DAST_VERSION: 1 + DAST_ZAP_CLI_OPTIONS: "-Xmx6144m -config database.recoverylog=false" #To avoid OutOfMemoryError error + DAST_EXCLUDE_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019" + script: + - apk add jq curl + - REVIEW_APPS_ROOT_TOKEN=$(cat environment_token.txt) + - CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - source scripts/review_apps/review-apps.sh + - disable_2fa_allgroup + needs: + # - job: DAST-prepare + # artifacts: false + - job: review-deploy + artifacts: true + stage: sec-dast + timeout: 2h + artifacts: + paths: + - gl-dast-report.json # GitLab-specific + reports: + dast: gl-dast-report.json + expire_in: 1 week # GitLab-specific +# DAST-fullscan-1/2 - This job runs the DAST scan with Release rule1/2 +DAST-fullscan-1/5: + extends: + - .dast_conf + resource_group: dast_scan + variables: + # DAST_USERNAME: "user1" + DAST_EXCLUDE_RULES: "$DAST_EXCLUDE_RULES,0,2,3,6,7,10010,10011,10015,10017,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90019,90020,90022,90033,41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028" + script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # - 'mkdir -p /zap/wrk' + - 'mkdir -p /zap/xml' + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - 'cat /zap/xml/config.xml' + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/groups/*/-/edit,${DAST_WEBSITE}/profile/two_factor_auth"' + - '/analyze -t $DAST_WEBSITE -d' +# DAST-fullscan-1/2 - This job runs the DAST scan with Release rule2/2 +DAST-fullscan-2/5: + extends: + - .dast_conf + resource_group: dast_scan + variables: + # DAST_USERNAME: "user2" + DAST_EXCLUDE_RULES: "$DAST_EXCLUDE_RULES,90019,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028" + script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # - 'mkdir -p /zap/wrk' + - 'mkdir -p /zap/xml' + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - 'cat /zap/xml/config.xml' + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/groups/*/-/edit,${DAST_WEBSITE}/profile/two_factor_auth"' + - '/analyze -t $DAST_WEBSITE -d' +# DAST-fullscan-1/2 - This job runs the DAST scan with Release rule2/2 +DAST-fullscan-3/5: + extends: + - .dast_conf + resource_group: dast_scan + variables: + # DAST_USERNAME: "user3" + DAST_EXCLUDE_RULES: "$DAST_EXCLUDE_RULES,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90019,90020,90022,90033,41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028" + script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # - 'mkdir -p /zap/wrk' + - 'mkdir -p /zap/xml' + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - 'cat /zap/xml/config.xml' + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/groups/*/-/edit,${DAST_WEBSITE}/profile/two_factor_auth"' + - '/analyze -t $DAST_WEBSITE -d' +# DAST-fullscan-1/2 - This job runs the DAST scan with Release rule2/2 +DAST-fullscan-4/5: + extends: + - .dast_conf + resource_group: dast_scan + variables: + # DAST_USERNAME: "user4" + DAST_EXCLUDE_RULES: "$DAST_EXCLUDE_RULES,40016,40017,40018,50000,50001,90011,90019,90020,90022,90033,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028" + script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # - 'mkdir -p /zap/wrk' + - 'mkdir -p /zap/xml' + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - 'cat /zap/xml/config.xml' + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/groups/*/-/edit,${DAST_WEBSITE}/profile/two_factor_auth"' + - '/analyze -t $DAST_WEBSITE -d' + +# DAST-fullscan-2/2 - This job runs the DAST scan with BETA rule +DAST-fullscan-5/5: + extends: + - .dast_conf + resource_group: dast_scan + variables: + # DAST_USERNAME: "user5" + DAST_EXCLUDE_RULES: "$DAST_EXCLUDE_RULES,41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90019,90020,90022,90033" + script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # - 'mkdir -p /zap/wrk' + - 'mkdir -p /zap/xml' + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - 'cat /zap/xml/config.xml' + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/groups/*/-/edit,${DAST_WEBSITE}/profile/two_factor_auth"' + - '/analyze -t $DAST_WEBSITE -d' \ No newline at end of file diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 4e3a80372a651279bc0943813dd9a9234d0171f3..33040f0f69bef9e573d79a3a6e34f9ba6ca69af7 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -77,8 +77,10 @@ review-deploy: # to have to manually start the jobs in sequence, so we do it for them. - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"' + after_script: + - echo "${REVIEW_APPS_ROOT_TOKEN}" > environment_token.txt artifacts: - paths: [environment_url.txt] + paths: [environment_url.txt, environment_token.txt] expire_in: 2 days when: always @@ -148,6 +150,9 @@ review-stop: when: always review-qa-smoke: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .review-qa-base - .review:rules:review-qa-smoke @@ -155,6 +160,9 @@ review-qa-smoke: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" review-qa-all: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .review-qa-base - .review:rules:mr-only-manual @@ -165,6 +173,9 @@ review-qa-all: - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation review-performance: + rules: + - if: '$DAST_TEST == "true"' + when: never extends: - .default-retry - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 74291f6aef4c4e4097acfd52358d97f612097f87..d0a291705d4fd23344d7a2eb0e96dd42b0b8c2ca 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -134,6 +134,10 @@ function run_task() { kubectl exec -it --namespace "${namespace}" "${task_runner_pod}" -- gitlab-rails runner "${ruby_cmd}" } +function disable_2fa_allgroup() { + local ruby_cmd="Group.update_all(require_two_factor_authentication: false)" #todo + run_task "${ruby_cmd}" +} function disable_sign_ups() { if [ -z ${REVIEW_APPS_ROOT_TOKEN+x} ]; then echoerr "In order to protect Review Apps, REVIEW_APPS_ROOT_TOKEN variable must be set" diff --git a/scripts/review_apps/seed-dast-test-data.sh b/scripts/review_apps/seed-dast-test-data.sh new file mode 100644 index 0000000000000000000000000000000000000000..95482e8db6727da1846a986a22f4222fc55c305f --- /dev/null +++ b/scripts/review_apps/seed-dast-test-data.sh @@ -0,0 +1,32 @@ +function create_user() { + local user="${1}" + + # API details at https://docs.gitlab.com/ee/api/users.html#user-creation + # + # We set "can_create_group=false" because we don't want the DAST user to create groups. + # Otherwise, the DAST user likely creates a group and enables 2FA for all group members, + # which leads to the DAST scan getting "stuck" on the 2FA set up page. + # Once https://gitlab.com/gitlab-org/gitlab/-/issues/231447 is resolved, we can use + # DAST_AUTH_EXCLUDE_URLS instead to prevent DAST from enabling 2FA. + curl --silent --show-error --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" \ + --data "email=${user}@example.com" \ + --data "name=${user}" \ + --data "username=${user}" \ + --data "password=${REVIEW_APPS_ROOT_PASSWORD}" \ + --data "skip_confirmation=true" \ + --data "can_create_group=false" \ + "${CI_ENVIRONMENT_URL}/api/v4/users" > /tmp/user.json + + [[ "$TRACE" ]] && cat /tmp/user.json >&2 +function create_project_for_user() { + local userid="${1}" + + # API details at https://docs.gitlab.com/ee/api/projects.html#create-project-for-user + curl --silent --show-error --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" \ + --data "user_id=${userid}" \ + --data "name=awesome-test-project-${userid}" \ + --data "visibility=private" \ + "${CI_ENVIRONMENT_URL}/api/v4/projects/user/${userid}" > /tmp/project.json + + [[ "$TRACE" ]] && cat /tmp/project.json >&2 +}