diff --git a/.dockerignore b/.dockerignore index de1a0d22116066d5f4872d93ef8de782cf0a7f87..92a0cd60ad4abdfe6ab5f1e010242696e21c06d5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,10 +7,6 @@ *.* -# But don't ignore these specific directories and subdirectories with specific files: - -!/docker - # But don't ignore these specific files: !.clang-* @@ -20,10 +16,8 @@ !.hadolint.yaml !.license.tmpl !.yamllint -!conanfile.txt -!requirements.txt +!pyproject.toml -!**.Dockerfile !**.patch !**.py !**.sh @@ -31,6 +25,13 @@ # ...even if they are in subdirectories !*/ +# But don't ignore these specific directories and subdirectories with specific files: + +!/conan +!/conan/** + +# But ignore these specific directories and subdirectories with specific files: + .git/ .idea/ .venv/ diff --git a/.gitignore b/.gitignore index 64e307bd42a28328a4d0ef87cb12f2ed6e6371ad..2b718b111946ee2ee4f6e11c13fb83f6dff5ef13 100644 --- a/.gitignore +++ b/.gitignore @@ -7,12 +7,6 @@ *.* -# But don't ignore these specific directories and subdirectories with specific files: - -!/.gitlab -!/docker -!/docs - # But don't ignore these specific files: !.clang-* @@ -25,21 +19,29 @@ !.hadolint.yaml !.license.tmpl !.yamllint -!conanfile.txt !CONTRIBUTING.md !LICENSE +!pyproject.toml !README.md -!requirements.txt !**.Dockerfile !**.patch !**.py !**.rst !**.sh +!**.yml # ...even if they are in subdirectories !*/ +# But don't ignore these specific directories and subdirectories with specific files: + +!/conan +!/conan/** + +# But ignore these specific directories and subdirectories with specific files: + +/*.egg-info /.idea/ /.venv/ /build/ diff --git a/.gitlab/ci/docs/sphinx.yml b/.gitlab/ci/docs/sphinx.yml index 13b85b949643e5984ec3bfad54b8b1e36231ac2a..aa264124b9dbaf7276fa870ef8faab64138a16d8 100644 --- a/.gitlab/ci/docs/sphinx.yml +++ b/.gitlab/ci/docs/sphinx.yml @@ -46,11 +46,15 @@ report:docs: script: - mv build/* . - rm -rf build + environment: + name: sphinx-html-docs-$CI_COMMIT_REF_SLUG + url: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts/index.html + auto_stop_in: 1 day artifacts: when: always expire_in: 1 day - expose_as: 'HTML Documentation' - name: "HtmlDocs-$CI_COMMIT_REF_SLUG" + expose_as: 'Sphinx HTML Documentation' + name: "SphinxHtmlDocs-$CI_COMMIT_REF_SLUG" paths: - ./ diff --git a/.gitlab/ci/external/cpp.yml b/.gitlab/ci/external/cpp.yml index fd5a82a1b1b5c7cead6594d1c37852b6b9cc9ada..b114403b3d27510d3e33792425bf1c2c930e56d9 100644 --- a/.gitlab/ci/external/cpp.yml +++ b/.gitlab/ci/external/cpp.yml @@ -17,14 +17,16 @@ include: ref: main file: - '/.gitlab/ci/external/lite.yml' - # C++ files: - '/.gitlab/ci/lang/cpp/base.yml' + - '/.gitlab/ci/lang/cpp/build/base.yml' + - '/.gitlab/ci/lang/cpp/build/centos7_gcc.yml' + - '/.gitlab/ci/lang/cpp/build/centos8_gcc.yml' # discontinued + - '/.gitlab/ci/lang/cpp/build/ubuntu2004_gcc.yml' + - '/.gitlab/ci/lang/cpp/build/ubuntu2304_clang.yml' + - '/.gitlab/ci/lang/cpp/build/ubuntu2304_gcc.yml' + - '/.gitlab/ci/lang/cpp/build/windows.yml' - '/.gitlab/ci/lang/cpp/lint/clang_tools.yml' - '/.gitlab/ci/lang/cpp/lint/coverage.yml' - '/.gitlab/ci/lang/cpp/lint/cppcheck.yml' + - '/.gitlab/ci/lang/cpp/lint/iostream.yml' - '/.gitlab/ci/lang/cpp/lint/guardonce.yml' - - '/.gitlab/ci/lang/cpp/centos_clang.yml' - - '/.gitlab/ci/lang/cpp/centos_gcc.yml' - - '/.gitlab/ci/lang/cpp/ubuntu_clang.yml' - - '/.gitlab/ci/lang/cpp/ubuntu_gcc.yml' - - '/.gitlab/ci/lang/cpp/windows.yml' diff --git a/.gitlab/ci/images/base.yml b/.gitlab/ci/images/base.yml index db4bd66cb7651f6f0f35fa4705ffd1bced898ef0..d276333637864c5d8efcba00b5a7a51649965658 100644 --- a/.gitlab/ci/images/base.yml +++ b/.gitlab/ci/images/base.yml @@ -22,9 +22,9 @@ # DOCKER_BUILDKIT=1 environment variable for your GitLab CI job. .docker: - image: docker:19.03.12 + image: docker:24.0.0 services: - - docker:19.03.12-dind + - docker:24.0.0-dind dependencies: [] needs: [] before_script: diff --git a/.gitlab/ci/images/centos7.yml b/.gitlab/ci/images/centos7.yml index bc5bc8130103425b5602121d751aebc06836eb89..f86e3ffd1ae51108847bbd426a51dc35a4688841 100644 --- a/.gitlab/ci/images/centos7.yml +++ b/.gitlab/ci/images/centos7.yml @@ -17,7 +17,6 @@ - conan/conanfile.txt - docker/CentOS7.Dockerfile - scripts/conan/* - - scripts/os/install_files.sh - scripts/os/setup_centos7.sh - requirements.txt @@ -48,4 +47,3 @@ deploy:docker:centos7: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' changes: *centos7-image-changes - - !reference [.trigger-docker-on-linters-files-change, rules] diff --git a/.gitlab/ci/images/centos8.yml b/.gitlab/ci/images/centos8.yml index 8708d918b3b8fbb1a8d301dcef1e4474fa56da63..ecb8edad03ff949905e13e886143927f861fe627 100644 --- a/.gitlab/ci/images/centos8.yml +++ b/.gitlab/ci/images/centos8.yml @@ -17,7 +17,6 @@ - conan/conanfile.txt - docker/CentOS8.Dockerfile - scripts/conan/* - - scripts/os/install_files.sh - scripts/os/setup_centos8.sh - requirements.txt @@ -48,4 +47,3 @@ deploy:docker:centos8: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' changes: *centos8-image-changes - - !reference [.trigger-docker-on-linters-files-change, rules] diff --git a/.gitlab/ci/images/ubuntu2004.yml b/.gitlab/ci/images/ubuntu2004.yml index 2d73884eb08cf3a0a9be02f862d0dec33a48d100..b5acb9b191e3933aa7a53720af9b9d8b41d14a8c 100644 --- a/.gitlab/ci/images/ubuntu2004.yml +++ b/.gitlab/ci/images/ubuntu2004.yml @@ -17,7 +17,6 @@ - conan/conanfile.txt - docker/Ubuntu2004.Dockerfile - scripts/conan/* - - scripts/os/install_files.sh - scripts/os/setup_ubuntu2004.sh - requirements.txt @@ -48,4 +47,3 @@ deploy:docker:ubuntu2004: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' changes: *ubuntu2004_changes - - !reference [.trigger-docker-on-linters-files-change, rules] diff --git a/.gitlab/ci/images/ubuntu2304.yml b/.gitlab/ci/images/ubuntu2304.yml new file mode 100644 index 0000000000000000000000000000000000000000..f088a924fa6bc950510758d2ccbb81e60a06c244 --- /dev/null +++ b/.gitlab/ci/images/ubuntu2304.yml @@ -0,0 +1,51 @@ +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.ubuntu2304_changes: &ubuntu2304_changes + - conan/profiles/linux_* + - conan/conanfile.txt + - docker/Ubuntu2304.Dockerfile + - scripts/conan/* + - scripts/os/install_files.sh + - scripts/os/setup_ubuntu2304.sh + - requirements.txt + +.ubuntu2304_variables: &ubuntu2304_variables + DOCKER_IMAGE_NAME: "ubuntu2304:latest" + DOCKER_IMAGE_OS: "Ubuntu2304" + +build:docker:ubuntu2304: + stage: build + extends: .docker + variables: + <<: *ubuntu2304_variables + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + *ubuntu2304_changes + +deploy:docker:ubuntu2304: + stage: deploy + extends: .docker + variables: + <<: *ubuntu2304_variables + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + changes: + *ubuntu2304_changes + - !reference [.trigger-docker-on-linters-files-change, rules] diff --git a/.gitlab/ci/index.yml b/.gitlab/ci/index.yml index f5ebadf0f174e482129486215aac14438299cc7e..13fecfd4b11c84541b6bb84c97254e8f90998c1b 100644 --- a/.gitlab/ci/index.yml +++ b/.gitlab/ci/index.yml @@ -24,21 +24,25 @@ include: - local: '/.gitlab/ci/images/conan.yml' - local: '/.gitlab/ci/images/hadolint.yml' - local: '/.gitlab/ci/images/ubuntu2004.yml' + - local: '/.gitlab/ci/images/ubuntu2304.yml' - local: '/.gitlab/ci/lang/cmake/base.yml' - local: '/.gitlab/ci/lang/cmake/lint/cmakelang.yml' # - local: '/.gitlab/ci/lang/common/lint/codequality.yml' - local: '/.gitlab/ci/lang/common/lint/codespell.yml' - local: '/.gitlab/ci/lang/common/lint/license_headers.yml' - local: '/.gitlab/ci/lang/cpp/base.yml' + - local: '/.gitlab/ci/lang/cpp/build/base.yml' + - local: '/.gitlab/ci/lang/cpp/build/centos7_gcc.yml' + - local: '/.gitlab/ci/lang/cpp/build/centos8_gcc.yml' + - local: '/.gitlab/ci/lang/cpp/build/ubuntu2004_gcc.yml' + - local: '/.gitlab/ci/lang/cpp/build/ubuntu2304_clang.yml' + - local: '/.gitlab/ci/lang/cpp/build/ubuntu2304_gcc.yml' + - local: '/.gitlab/ci/lang/cpp/build/windows.yml' - local: '/.gitlab/ci/lang/cpp/lint/clang_tools.yml' - local: '/.gitlab/ci/lang/cpp/lint/coverage.yml' - local: '/.gitlab/ci/lang/cpp/lint/cppcheck.yml' + - local: '/.gitlab/ci/lang/cpp/lint/iostream.yml' - local: '/.gitlab/ci/lang/cpp/lint/guardonce.yml' - - local: '/.gitlab/ci/lang/cpp/centos_clang.yml' - - local: '/.gitlab/ci/lang/cpp/centos_gcc.yml' - - local: '/.gitlab/ci/lang/cpp/ubuntu_clang.yml' - - local: '/.gitlab/ci/lang/cpp/ubuntu_gcc.yml' - - local: '/.gitlab/ci/lang/cpp/windows.yml' - local: '/.gitlab/ci/lang/docker/lint/hadolint.yml' - local: '/.gitlab/ci/lang/py/base.yml' - local: '/.gitlab/ci/lang/py/lint/black.yml' diff --git a/.gitlab/ci/lang/cpp/base.yml b/.gitlab/ci/lang/cpp/base.yml index 7c57f4b7d7791f98ff8ebf66c20ebefb858155ff..bfc58c0ab3b8e9e00329c11bc936d08148ddca2c 100644 --- a/.gitlab/ci/lang/cpp/base.yml +++ b/.gitlab/ci/lang/cpp/base.yml @@ -23,6 +23,25 @@ - "**/*.hpp" - "**/*.ipp" +.only-review-cpp-manual: + only: + rules: + - if: '$CI_PROJECT_NAME == "cicd"' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_PIPELINE_SOURCE == "web"' + when: manual + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + - "**/CMakeLists.txt" + - "**/*.cmake" + - "**/*.cpp" + - "**/*.hpp" + - "**/*.ipp" + when: manual + allow_failure: true + .only-review-hpp: only: refs: @@ -30,135 +49,3 @@ changes: - "**/*.hpp" - "**/*.ipp" - -# Build - -.build-cpp: - stage: build - extends: .only-review-cpp - interruptible: false - variables: - # Use prefixed CMake options to avoid UB while executing 'conan install'. - BPROTO_CI_CMAKE_BUILD_TYPE: "Debug" - # This can be defined global wide. - CMAKE_GENERATOR: Unix Makefiles - before_script: - # nproc on GNU/Linux, sysctl on macOS - - NPROC=$(nproc || sysctl -n hw.physicalcpu) 2>/dev/null - # Download all Conan packages - - |- - if [[ -f "${CI_PROJECT_DIR}/conanfile.py" || -f "${CI_PROJECT_DIR}/conanfile.txt" ]]; then - conan install . -if build -of build --build=missing \ - -pr:b "${CC}" \ - -pr:h "${CC}" \ - -s:b build_type="${BPROTO_CI_CMAKE_BUILD_TYPE}" \ - -s:h build_type="${BPROTO_CI_CMAKE_BUILD_TYPE}" - - # Conan will generate this for us. - export BPROTO_CI_CMAKE_TOOLCHAIN_FILE="-DCMAKE_TOOLCHAIN_FILE='conan_toolchain.cmake'" - fi - - export BPROTO_CI_CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=${BPROTO_CI_CMAKE_BUILD_TYPE}" - - |- - echo -e "BPROTO_CI_CMAKE_BUILD_TYPE:\n\t${BPROTO_CI_CMAKE_BUILD_TYPE}" - echo -e "BPROTO_CI_CMAKE_ARGS:\n\t${BPROTO_CI_CMAKE_ARGS}" - echo -e "BPROTO_CI_CMAKE_EXTRA_ARGS:\n\t${BPROTO_CI_CMAKE_EXTRA_ARGS}" - echo -e "BPROTO_CI_CMAKE_TOOLCHAIN_FILE:\n\t${BPROTO_CI_CMAKE_TOOLCHAIN_FILE}" - script: - - mkdir -p build - - |- - cmake -S. -Bbuild \ - ${BPROTO_CI_CMAKE_BUILD_TYPE} \ - ${BPROTO_CI_CMAKE_ARGS} \ - ${BPROTO_CI_CMAKE_EXTRA_ARGS} \ - ${BPROTO_CI_CMAKE_TOOLCHAIN_FILE} \ - 2>&1 | tee build/ConfigurationLog.txt - - cmake --build build -j${NPROC} 2>&1 | tee build/CompilationLog.txt - artifacts: - expire_in: 2 hrs - when: always - paths: - # Different logs are needed if something happens. - - build/CMakeFiles/CMakeError.log - - build/CMakeFiles/CMakeOutput.log - - build/ConfigurationLog.txt - - build/CompilationLog.txt - - # The main binaries. - - build/bin/ - - # The cache is needed for the installation test. - - build/CMakeCache.txt - - # CTest files are needed for the installation test. - - build/CTestTestfile.cmake - - build/**/CTestTestfile.cmake - - # Dart files are needed for the tests with valgrind. - - build/DartConfiguration.tcl - - # Test stage doesn't checkout repo and use only artifacts. - # So include test configs to an artifact too. - - tools/* - -# Test - -.test-cpp: - stage: test - extends: .only-review-cpp - when: on_success - variables: - # Boost settings - BOOST_TEST_LOGGER: "JUNIT,all,test_results.xml:HRF,warning" - # CTest settings - CTEST_OUTPUT_ON_FAILURE: 1 - # GIT settings - GIT_CHECKOUT: "false" - GIT_STRATEGY: none - # Sanitizers settings - ASAN_OPTIONS: "log_threads=1:verbosity=1" - LSAN_OPTIONS: "log_threads=1:verbosity=1" - TSAN_OPTIONS: "log_threads=1:verbosity=1" - before_script: - # nproc on GNU/Linux, sysctl on macOS - - NPROC=$(nproc || sysctl -n hw.physicalcpu) 2>/dev/null - - |- - asan_supp_file="${CI_PROJECT_DIR}/tools/asan.supp" - if [[ -f "${asan_supp_file}" ]]; then - export ASAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${asan_supp_file}" - fi - - |- - lsan_supp_file="${CI_PROJECT_DIR}/tools/lsan.supp" - if [[ -f "${lsan_supp_file}" ]]; then - export LSAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${lsan_supp_file}" - fi - - |- - tsan_supp_file="${CI_PROJECT_DIR}/tools/tsan.supp" - if [[ -f "${tsan_supp_file}" ]]; then - export TSAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${tsan_supp_file}" - fi - script: - - ctest -j${NPROC} --test-dir build - artifacts: - reports: - junit: - - build/**/test_results.xml - -# Report - -# If build job failed, upload compilation logs in an easily downloadable form. -.report-build-cpp: - extends: - - .only-review-cpp - - .report - script: - - cp /build/CMakeFiles/CMakeError.log . - - cp /build/CMakeFiles/CMakeOutput.log . - artifacts: - expire_in: 1 day - expose_as: 'Build log' - name: "BuildLog-$CI_COMMIT_REF_SLUG" - paths: - - CMakeError.log - - CMakeOutput.log - - ConfigurationLog.txt - - CompilationLog.txt diff --git a/.gitlab/ci/lang/cpp/build/base.yml b/.gitlab/ci/lang/cpp/build/base.yml new file mode 100644 index 0000000000000000000000000000000000000000..1bb3064edeacfa619545a0f602bb5ee4c2b60741 --- /dev/null +++ b/.gitlab/ci/lang/cpp/build/base.yml @@ -0,0 +1,139 @@ +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.build-cpp: + stage: build + extends: .only-review-cpp + interruptible: false + variables: + # Use prefixed CMake options to avoid UB while executing 'conan install'. + BPROTO_CI_CMAKE_BUILD_TYPE: "Debug" + # This can be defined global wide. + CMAKE_GENERATOR: Unix Makefiles + before_script: + # nproc on GNU/Linux, sysctl on macOS + - NPROC=$(nproc || sysctl -n hw.physicalcpu) 2>/dev/null + # Download all Conan packages + - |- + if [[ -f "${CI_PROJECT_DIR}/conanfile.py" || -f "${CI_PROJECT_DIR}/conanfile.txt" ]]; then + conan install . -if build -of build --build=missing \ + -pr:b "${CC}" \ + -pr:h "${CC}" \ + -s:b build_type="${BPROTO_CI_CMAKE_BUILD_TYPE}" \ + -s:h build_type="${BPROTO_CI_CMAKE_BUILD_TYPE}" + + # Conan will generate this for us. + export BPROTO_CI_CMAKE_TOOLCHAIN_FILE="-DCMAKE_TOOLCHAIN_FILE='conan_toolchain.cmake'" + fi + - export BPROTO_CI_CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=${BPROTO_CI_CMAKE_BUILD_TYPE}" + - |- + echo -e "BPROTO_CI_CMAKE_BUILD_TYPE:\n\t${BPROTO_CI_CMAKE_BUILD_TYPE}" + echo -e "BPROTO_CI_CMAKE_ARGS:\n\t${BPROTO_CI_CMAKE_ARGS}" + echo -e "BPROTO_CI_CMAKE_EXTRA_ARGS:\n\t${BPROTO_CI_CMAKE_EXTRA_ARGS}" + echo -e "BPROTO_CI_CMAKE_TOOLCHAIN_FILE:\n\t${BPROTO_CI_CMAKE_TOOLCHAIN_FILE}" + script: + - mkdir -p build + - |- + cmake -S. -Bbuild \ + ${BPROTO_CI_CMAKE_BUILD_TYPE} \ + ${BPROTO_CI_CMAKE_ARGS} \ + ${BPROTO_CI_CMAKE_EXTRA_ARGS} \ + ${BPROTO_CI_CMAKE_TOOLCHAIN_FILE} \ + 2>&1 | tee build/ConfigurationLog.txt + - cmake --build build -j${NPROC} 2>&1 | tee build/CompilationLog.txt + artifacts: + expire_in: 2 hrs + when: always + paths: + # Different logs are needed if something happens. + - build/CMakeFiles/CMakeError.log + - build/CMakeFiles/CMakeOutput.log + - build/ConfigurationLog.txt + - build/CompilationLog.txt + + # The main binaries. + - build/bin/ + + # The cache is needed for the installation test. + - build/CMakeCache.txt + + # CTest files are needed for the installation test. + - build/CTestTestfile.cmake + - build/**/CTestTestfile.cmake + + # Dart files are needed for the tests with valgrind. + - build/DartConfiguration.tcl + + # Test stage doesn't checkout repo and use only artifacts. + # So include test configs to an artifact too. + - tools/* + +.test-cpp: + stage: test + extends: .only-review-cpp + when: on_success + variables: + # Boost settings + BOOST_TEST_LOGGER: "JUNIT,all,test_results.xml:HRF,warning" + # CTest settings + CTEST_OUTPUT_ON_FAILURE: 1 + # GIT settings + GIT_CHECKOUT: "false" + GIT_STRATEGY: none + # Sanitizers settings + ASAN_OPTIONS: "log_threads=1:verbosity=1" + LSAN_OPTIONS: "log_threads=1:verbosity=1" + TSAN_OPTIONS: "log_threads=1:verbosity=1" + before_script: + # nproc on GNU/Linux, sysctl on macOS + - NPROC=$(nproc || sysctl -n hw.physicalcpu) 2>/dev/null + - |- + asan_supp_file="${CI_PROJECT_DIR}/tools/asan.supp" + if [[ -f "${asan_supp_file}" ]]; then + export ASAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${asan_supp_file}" + fi + - |- + lsan_supp_file="${CI_PROJECT_DIR}/tools/lsan.supp" + if [[ -f "${lsan_supp_file}" ]]; then + export LSAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${lsan_supp_file}" + fi + - |- + tsan_supp_file="${CI_PROJECT_DIR}/tools/tsan.supp" + if [[ -f "${tsan_supp_file}" ]]; then + export TSAN_OPTIONS="${ASAN_OPTIONS}:suppressions=${tsan_supp_file}" + fi + script: + - ctest -j${NPROC} --test-dir build + artifacts: + reports: + junit: + - build/**/test_results.xml + +# If build job failed, upload compilation logs in an easily downloadable form. +.report-build-cpp: + extends: + - .only-review-cpp + - .report + script: + - cp /build/CMakeFiles/CMakeError.log . + - cp /build/CMakeFiles/CMakeOutput.log . + artifacts: + expire_in: 1 day + expose_as: 'Build log' + name: "BuildLog-$CI_COMMIT_REF_SLUG" + paths: + - CMakeError.log + - CMakeOutput.log + - ConfigurationLog.txt + - CompilationLog.txt diff --git a/.gitlab/ci/lang/cpp/centos_gcc.yml b/.gitlab/ci/lang/cpp/build/centos7_gcc.yml similarity index 60% rename from .gitlab/ci/lang/cpp/centos_gcc.yml rename to .gitlab/ci/lang/cpp/build/centos7_gcc.yml index b87661b594f2d012eb02a0383f3af1e0e3732988..00f5f29451b4e2fdfc5d0808001e0d3c9720b564 100644 --- a/.gitlab/ci/lang/cpp/centos_gcc.yml +++ b/.gitlab/ci/lang/cpp/build/centos7_gcc.yml @@ -19,21 +19,7 @@ build:cpp:centos7:gcc: extends: .build-cpp variables: BPROTO_CI_CMAKE_BUILD_TYPE: "Release" - BPROTO_CI_CMAKE_ARGS: >- - -DCMOPTS_ENABLE_LTO=ON - -DBPROTO_APPEND_VC_REV=ON - CC: gcc - CXX: g++ - -build:cpp:centos8:gcc: - image: $BPROTO_CI_IMAGES_REGISTRY/centos8:latest - extends: .build-cpp - variables: - BPROTO_CI_CMAKE_ARGS: >- - -DCMOPTS_ENABLE_LTO=OFF - -DCMOPTS_ENABLE_SANITIZER_ADDRESS=ON - -DCMOPTS_ENABLE_SANITIZER_LEAK=ON - -DCMOPTS_ENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON + BPROTO_CI_CMAKE_ARGS: "-DCMOPTS_ENABLE_LTO=ON" CC: gcc CXX: g++ @@ -46,13 +32,6 @@ test:cpp:centos7:gcc: - job: build:cpp:centos7:gcc artifacts: true -test:cpp:centos8:gcc: - image: $BPROTO_CI_IMAGES_REGISTRY/centos8:latest - extends: .test-cpp - needs: - - job: build:cpp:centos8:gcc - artifacts: true - # Report report:cpp:centos7:gcc: @@ -60,9 +39,3 @@ report:cpp:centos7:gcc: needs: - job: build:cpp:centos7:gcc artifacts: true - -report:cpp:centos8:gcc: - extends: .report-build-cpp - needs: - - job: build:cpp:centos8:gcc - artifacts: true diff --git a/.gitlab/ci/lang/cpp/centos_clang.yml b/.gitlab/ci/lang/cpp/build/centos8_gcc.yml similarity index 54% rename from .gitlab/ci/lang/cpp/centos_clang.yml rename to .gitlab/ci/lang/cpp/build/centos8_gcc.yml index da370e2ff0cdab9b014908d222f08f7c43fe8265..27cea9951c22cb11ccfda21ad04fd3a6404c049c 100644 --- a/.gitlab/ci/lang/cpp/centos_clang.yml +++ b/.gitlab/ci/lang/cpp/build/centos8_gcc.yml @@ -14,48 +14,30 @@ # Build -build:cpp:centos7:clang: - image: $BPROTO_CI_IMAGES_REGISTRY/centos7:latest - extends: .build-cpp - variables: - BPROTO_CI_CMAKE_ARGS: "-DCMOPTS_ENABLE_LTO=OFF" - CC: clang - CXX: clang++ - -build:cpp:centos8:clang: +build:cpp:centos8:gcc: image: $BPROTO_CI_IMAGES_REGISTRY/centos8:latest - extends: .build-cpp + extends: + - .build-cpp + - .only-review-cpp-manual variables: - BPROTO_CI_CMAKE_ARGS: -DCMOPTS_ENABLE_LTO=OFF" - CC: clang - CXX: clang++ + BPROTO_CI_CMAKE_BUILD_TYPE: "Release" + BPROTO_CI_CMAKE_ARGS: "-DCMOPTS_ENABLE_LTO=ON" + CC: gcc + CXX: g++ # Test -test:cpp:centos7:clang: - image: $BPROTO_CI_IMAGES_REGISTRY/centos7:latest - extends: .test-cpp - needs: - - job: build:cpp:centos7:clang - artifacts: true - -test:cpp:centos8:clang: +test:cpp:centos8:gcc: image: $BPROTO_CI_IMAGES_REGISTRY/centos8:latest extends: .test-cpp needs: - - job: build:cpp:centos8:clang + - job: build:cpp:centos8:gcc artifacts: true # Report -report:cpp:centos7:clang: - extends: .report-build-cpp - needs: - - job: build:cpp:centos7:clang - artifacts: true - -report:cpp:centos8:clang: +report:cpp:centos8:gcc: extends: .report-build-cpp needs: - - job: build:cpp:centos8:clang + - job: build:cpp:centos8:gcc artifacts: true diff --git a/.gitlab/ci/lang/cpp/build/ubuntu2004_gcc.yml b/.gitlab/ci/lang/cpp/build/ubuntu2004_gcc.yml new file mode 100644 index 0000000000000000000000000000000000000000..d454f6c39139fa701ebb4b780223f5e89e02a8b8 --- /dev/null +++ b/.gitlab/ci/lang/cpp/build/ubuntu2004_gcc.yml @@ -0,0 +1,43 @@ +# Copyright 2020-2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build + +build:cpp:ubuntu2004:gcc: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest + extends: + - .build-cpp + - .only-review-cpp-manual + variables: + BPROTO_CI_CMAKE_BUILD_TYPE: "Release" + BPROTO_CI_CMAKE_ARGS: "-DCMOPTS_ENABLE_LTO=ON" + CC: gcc + CXX: g++ + +# Test + +test:cpp:ubuntu2004:gcc: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest + extends: .test-cpp + needs: + - job: build:cpp:ubuntu2004:gcc + artifacts: true + +# Report + +report:cpp:ubuntu2004:gcc: + extends: .report-build-cpp + needs: + - job: build:cpp:ubuntu2004:gcc + artifacts: true diff --git a/.gitlab/ci/lang/cpp/ubuntu_clang.yml b/.gitlab/ci/lang/cpp/build/ubuntu2304_clang.yml similarity index 72% rename from .gitlab/ci/lang/cpp/ubuntu_clang.yml rename to .gitlab/ci/lang/cpp/build/ubuntu2304_clang.yml index b02c722800a8fcb1be0dfa0552ad10d75686310b..1a8bb3686986d6beb94d7b2fde6b0ae2d409eabe 100644 --- a/.gitlab/ci/lang/cpp/ubuntu_clang.yml +++ b/.gitlab/ci/lang/cpp/build/ubuntu2304_clang.yml @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Zinchenko Serhii . +# Copyright 2023 Zinchenko Serhii . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,30 +14,30 @@ # Build -build:cpp:ubuntu2004:clang: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest +build:cpp:ubuntu2304:clang: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest extends: .build-cpp variables: BPROTO_CI_CMAKE_ARGS: >- - -DCPPDOC_ENABLE=ON -DCMOPTS_ENABLE_LTO=OFF -DCMOPTS_ENABLE_SANITIZER_THREAD=ON + -DCPPDOC_ENABLE=ON CC: clang CXX: clang++ # Test -test:cpp:ubuntu2004:clang: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest +test:cpp:ubuntu2304:clang: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest extends: .test-cpp needs: - - job: build:cpp:ubuntu2004:clang + - job: build:cpp:ubuntu2304:clang artifacts: true # Report -report:cpp:ubuntu2004:clang: +report:cpp:ubuntu2304:clang: extends: .report-build-cpp needs: - - job: build:cpp:ubuntu2004:clang + - job: build:cpp:ubuntu2304:clang artifacts: true diff --git a/.gitlab/ci/lang/cpp/ubuntu_gcc.yml b/.gitlab/ci/lang/cpp/build/ubuntu2304_gcc.yml similarity index 60% rename from .gitlab/ci/lang/cpp/ubuntu_gcc.yml rename to .gitlab/ci/lang/cpp/build/ubuntu2304_gcc.yml index e8933aa75d9ec832986aa95a7796ddb84be30be7..458c95f947fc34876000683784b5a30603894014 100644 --- a/.gitlab/ci/lang/cpp/ubuntu_gcc.yml +++ b/.gitlab/ci/lang/cpp/build/ubuntu2304_gcc.yml @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Zinchenko Serhii . +# Copyright 2023 Zinchenko Serhii . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,20 @@ # Build -build:cpp:ubuntu2004:gcc: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest +build:cpp:ubuntu2304:gcc:1: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest + extends: .build-cpp + variables: + BPROTO_CI_CMAKE_ARGS: >- + -DCMOPTS_ENABLE_LTO=OFF + -DCMOPTS_ENABLE_SANITIZER_ADDRESS=ON + -DCMOPTS_ENABLE_SANITIZER_LEAK=ON + -DCMOPTS_ENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON + CC: gcc + CXX: g++ + +build:cpp:ubuntu2304:gcc:2: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest extends: .build-cpp variables: BPROTO_CI_CMAKE_ARGS: >- @@ -26,11 +38,18 @@ build:cpp:ubuntu2004:gcc: # Test -test:cpp:ubuntu2004:gcc: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest +test:cpp:ubuntu2304:gcc:1: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest + extends: .test-cpp + needs: + - job: build:cpp:ubuntu2304:gcc:1 + artifacts: true + +test:cpp:ubuntu2304:gcc:2: + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest extends: .test-cpp needs: - - job: build:cpp:ubuntu2004:gcc + - job: build:cpp:ubuntu2304:gcc:2 artifacts: true script: - cd build @@ -56,8 +75,14 @@ test:cpp:ubuntu2004:gcc: # Report -report:cpp:ubuntu2004:gcc: +report:cpp:ubuntu2304:gcc:1: + extends: .report-build-cpp + needs: + - job: build:cpp:ubuntu2304:gcc:1 + artifacts: true + +report:cpp:ubuntu2304:gcc:2: extends: .report-build-cpp needs: - - job: build:cpp:ubuntu2004:gcc + - job: build:cpp:ubuntu2304:gcc:2 artifacts: true diff --git a/.gitlab/ci/lang/cpp/windows.yml b/.gitlab/ci/lang/cpp/build/windows.yml similarity index 85% rename from .gitlab/ci/lang/cpp/windows.yml rename to .gitlab/ci/lang/cpp/build/windows.yml index e31556ab7e3e57100aca1c7a53dc55f8ad93760a..d556543eb1a1cb37b9533e13dc16a08140ee8d86 100644 --- a/.gitlab/ci/lang/cpp/windows.yml +++ b/.gitlab/ci/lang/cpp/build/windows.yml @@ -32,7 +32,9 @@ build:cpp:windows: stage: build - extends: .windows-cpp-tags + extends: + - .only-review-cpp-manual + - .windows-cpp-tags variables: CACHE_DIR: "$CI_PROJECT_DIR\\.cache" # Setup package managers @@ -60,22 +62,6 @@ build:cpp:windows: cmake --build build -j"$env:NUMBER_OF_PROCESSORS" 2>&1 ` | % ToString | Tee-Object build/CompilationLog.txt } - rules: - - if: '$CI_PROJECT_NAME == "cicd"' - when: never - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$CI_PIPELINE_SOURCE == "web"' - when: manual - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - "**/CMakeLists.txt" - - "**/*.cmake" - - "**/*.cpp" - - "**/*.hpp" - - "**/*.ipp" - when: manual - allow_failure: true artifacts: !reference [.build-cpp, artifacts] diff --git a/.gitlab/ci/lang/cpp/lint/clang_tools.yml b/.gitlab/ci/lang/cpp/lint/clang_tools.yml index 7f69ff7c442b42008a6629a7857bb5c4941d381e..73f6450ea709cfebc5b4456985c9e876893c3841 100644 --- a/.gitlab/ci/lang/cpp/lint/clang_tools.yml +++ b/.gitlab/ci/lang/cpp/lint/clang_tools.yml @@ -28,7 +28,7 @@ check:cpp:format: - clang_format.patch .chck-cpp-clang-tidy: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest stage: check extends: - .build-cpp diff --git a/.gitlab/ci/lang/cpp/lint/coverage.yml b/.gitlab/ci/lang/cpp/lint/coverage.yml index 888944dacdaec605fad7699ae9210a3e6f8fef40..e555886ddda079d8d83ab05780b50dfa96f70cac 100644 --- a/.gitlab/ci/lang/cpp/lint/coverage.yml +++ b/.gitlab/ci/lang/cpp/lint/coverage.yml @@ -17,7 +17,7 @@ # It also worth to mention that such an artifact can occupy quite a lot of # space. check:cpp:coverage: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest stage: check extends: - .build-cpp @@ -57,6 +57,10 @@ report:cpp:coverage: script: - mv build/coverage/* . - rm -rf build + environment: + name: coverage-html-docs-$CI_COMMIT_REF_SLUG + url: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts/coverage.html + auto_stop_in: 1 day artifacts: when: always expire_in: 1 day diff --git a/.gitlab/ci/lang/cpp/lint/cppcheck.yml b/.gitlab/ci/lang/cpp/lint/cppcheck.yml index 299ea3319ee35e68d180dae038bf166f4d9e788a..8def54c8aa7333852ab7f07a7d3d98744bd02e53 100644 --- a/.gitlab/ci/lang/cpp/lint/cppcheck.yml +++ b/.gitlab/ci/lang/cpp/lint/cppcheck.yml @@ -13,7 +13,7 @@ # limitations under the License. check:cpp:cppcheck: - image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2004:latest + image: $BPROTO_CI_IMAGES_REGISTRY/ubuntu2304:latest stage: check extends: - .build-cpp @@ -52,6 +52,10 @@ report:cpp:cppcheck: script: - mv build/cppcheck_results/* . - rm -rf build + environment: + name: cppcheck-html-docs-$CI_COMMIT_REF_SLUG + url: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts/index.html + auto_stop_in: 1 day artifacts: when: always expire_in: 1 day diff --git a/.gitlab/ci/lang/cpp/lint/guardonce.yml b/.gitlab/ci/lang/cpp/lint/guardonce.yml index 044f0bba3abb4f7f053fcae3ab9f353bf25f4c93..c481e979cb4c4ba0e80fb3f58f90d8139b897292 100644 --- a/.gitlab/ci/lang/cpp/lint/guardonce.yml +++ b/.gitlab/ci/lang/cpp/lint/guardonce.yml @@ -17,7 +17,7 @@ check:cpp:include_guards: - .check - .only-review-hpp script: - - check_include_guards.sh + - check_cpp_include_guards.sh artifacts: when: on_failure expire_in: 1 day diff --git a/.gitlab/ci/lang/cpp/lint/iostream.yml b/.gitlab/ci/lang/cpp/lint/iostream.yml new file mode 100644 index 0000000000000000000000000000000000000000..d0e5b5147c311ff290e57d7e8e5f37b7f0d56efa --- /dev/null +++ b/.gitlab/ci/lang/cpp/lint/iostream.yml @@ -0,0 +1,20 @@ +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +check:cpp:iostream: + extends: + - .check + - .only-review-cpp + script: + - check_cpp_iostream.sh diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 03943f16305fc8378993574b80f223e2998ae4c2..794b3d58181f89ac2b055a4bc270472d72fc3737 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -2,8 +2,6 @@ // See http://go.microsoft.com/fwlink/?LinkId=827846 // for the documentation about the extensions.json format "recommendations": [ - // Format listfiles so they don't look like crap - "cheshirekow.cmake-format", // Markdown linting and style checking for Visual Studio Code "davidanson.vscode-markdownlint", // Supercharge the Git capabilities built into Visual Studio Code @@ -18,6 +16,8 @@ "ms-vscode.cmake-tools", // Spelling checker for source code "streetsidesoftware.code-spell-checker", + // Fully-featured TOML support + "tamasfe.even-better-toml", // CMake language support for Visual Studio Code "twxs.cmake", // Syntax highlighting and document symbols for reStructuredText diff --git a/.vscode/settings.json b/.vscode/settings.json index 73b63aaf7a37128df81c339b3efe099dc6f13eea..132a081575bce65b2dae729b796c429d77c8e37d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,4 @@ { - "[cmake]": { - "editor.defaultFormatter": "cheshirekow.cmake-format" - }, - "cmakeFormat.args": [ - "--config-file", - "${workspaceFolder}/.cmake-format.py" - ], "editor.bracketPairColorization.enabled": true, "editor.detectIndentation": false, "editor.formatOnPaste": false, @@ -23,21 +16,17 @@ "files.associations": { ".clang-tidy": "yaml", ".codespellrc": "ini", - "*.cpp.in": "cpp", - "*.ipp": "cpp", - "*.py.in": "python" }, "files.exclude": { ".git/**": true, "build*/**": true, - "cmake/build/": false, "**/*.vc.db*": true }, "files.insertFinalNewline": true, "files.trimTrailingWhitespace": true, "files.watcherExclude": { + ".git/**": true, "build*/**": true, - "cmake/build/": false, "**/*.vc.db*": true }, "gitlens.codeLens.recentChange.enabled": false, @@ -46,15 +35,9 @@ "search.exclude": { ".git/**": true, "build*/**": true, - "cmake/build/": false, "**/*.vc.db*": true }, - "theme-by-language.themes": { - "*": "Visual Studio Dark", - "c": "Visual Studio Dark - C++", - "cpp": "Visual Studio Dark - C++", - "python": "Default Dark+" - }, "workbench.editor.enablePreview": false, - "workbench.colorTheme": "Visual Studio Dark" + "workbench.colorTheme": "Visual Studio Dark", + "esbonio.sphinx.confDir": "" } diff --git a/README.md b/README.md index 1d9aed5e618d61ce40498cdd1b67d66cddcf6b5a..3aa605d53792dfb2cbe13411c8048653523835b8 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,80 @@ -# Cpp Template Projecte - Common CI/CD Tools +# GitLab Common CI/CD Tools -This repository contains content for our CI/CD. +[fork]: https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html -## Contributing +**GitLab Common CI/CD Tools** (**GitLab 3CT** for short) provides a [GitLab CI/CD](https://docs.gitlab.com/ee/ci) solution that helps to control the quality of mainly C++ and Python codebases and their surrounding infrastructure. -1. Clone this repository: +While the driving force of this module is to reduce duplication of CI/CD configurations and scripts across projects developed by me ([Zinchenko Serhii](https://gitlab.com/b110011)), it is intended to be useful for any software that wants to utilize GitLab CI/CD to some degree. -```bash -$ mkdir bproto -$ cd bproto -$ git clone https://gitlab.com/bproto/cicd.git -$ cd cicd -``` +This module also may serve as a useful reference for setting up a GitLab CI/CD using good practices. -You can also clone this repository using [GitLab token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). +**Table of contents** (generated with [markdown-toc](http://ecotrust-canada.github.io/markdown-toc/)) -```bash -$ git clone https://oauth2:@gitlab.com/bproto/cicd.git -``` +- [Usage](#usage) + - [CI/CD templates](#cicd-templates) + - [Forks](#forks) +- [License](#license) + +## Usage + +By design this project is intended to be used as a [CI/CD template](https://docs.gitlab.com/ee/ci/examples/index.html) in most of the cases. +However due to some limitations it might be [forked][fork] to bring some changes and then to be used as your own CI/CD template. +Both ways will be covered in-deep down on this page. + +### CI/CD templates + +This project provides a bunch of useful [GitLab CI/CD pipelines](https://docs.gitlab.com/ee/ci/pipelines/index.html) for different programming languages and frameworks. +Below is a nice list of all available pipelines: + +[conan-tmpl]: .gitlab/ci/external/conan.yml +[cpp-tmpl]: .gitlab/ci/external/cpp.yml +[hugo-tmpl]: .gitlab/ci/external/hugo.yml +[lite-tmpl]: .gitlab/ci/external/conan.yml +[sphinx-tmpl]: .gitlab/ci/external/sphinx.yml -*For alternative ways you can check out [this SO thread](https://stackoverflow.com/questions/25409700).* +- **Documentation** + - Hugo ([.gitlab/ci/external/hugo.yml][hugo-tmpl]) + - Sphinx ([.gitlab/ci/external/sphinx.yml][sphinx-tmpl]) +- **Languages** + - C++ ([.gitlab/ci/external/cpp.yml][cpp-tmpl]) + - CMake ([.gitlab/ci/external/lite.yml][lite-tmpl]) + - Docker ([.gitlab/ci/external/lite.yml][lite-tmpl]) + - Linux shell ([.gitlab/ci/external/lite.yml][lite-tmpl]) + - Python ([.gitlab/ci/external/lite.yml][lite-tmpl]) + - YAML ([.gitlab/ci/external/lite.yml][lite-tmpl]) +- **Packages** + - Conan ([.gitlab/ci/external/conan.yml][conan-tmpl]) -2. Setup a default merge strategy for pulling branches. +You can start using those pipelines by including one of them as shown below: -```bash -$ git config pull.rebase true +```yaml +include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' ``` -3. Create a branch. - This can either be an informal user branch or a full-fledged ticket branch tracked in JIRA. +You also need to define below [CI/CD variables](https://docs.gitlab.com/ee/ci/variables) in your [group](https://docs.gitlab.com/ee/ci/variables/#for-a-group) or [project](https://docs.gitlab.com/ee/ci/variables/#for-a-project) CI/CD settings. -4. Make and commit your edits. +| Type | Key | Value | Protected | Masked | Environments | +| --- | --- | --- | --- | --- | --- | +| Variable | BPROTO_CI_IMAGES_REGISTRY | `registry.gitlab.com/bproto/artifacts` | no | no | All | -5. Push your development branch to GitLab and make a merge request. - The merge request page will help you track the publishing and testing status of your branch. +Now you are ready to go! -6. Once you've passed a code review, ask of repository maintainers to merge your changes to `master` branch. +[overview]: https://bproto.gitlab.io/cicd/overview/index.html +[overview-stages]: https://bproto.gitlab.io/cicd/overview/stages.html +[overview-workflow]: https://bproto.gitlab.io/cicd/overview/workflow.html -## Prepearing development environment +> To have a better understanding of pipelines implementation, you might also consider looking at an [overview][overview] of different aspects of pipelines, such as a list of implemented [stages][overview-stages] and how the [workflow][overview-workflow] was designed. -*Assuming you are using of Ubuntu based distributions.* +### Forks -### Installing your favorite code editor +[forking]: https://bproto.gitlab.io/cicd/overview/index.html -Please consult our [documentation](https://bproto.gitlab.io/editors/index.html) for configuration suggestions of your favorite code editor. +If you think, you need to fork this project and adjust it to your personal needs, please consider taking a look at [this documentation page][forking]. -## Editing entirely on GitLab +## License -If you're in a hurry, you don't need to worry about cloning the Developer Guide; you can do everything on GitLab.com. -See [GitLab's documentation](https://docs.gitlab.com/ee/user/project/web_ide/) on editing files and creating branches entirely from [GitLab.com](https://gitlab.com). +All code is licensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. diff --git a/docker/Alpine3.Dockerfile b/docker/Alpine3.Dockerfile index aa401a23636eb6e3af30728cb275e8ddf47b09a1..76bcca9134b646c06995869599bcd3333077cd5f 100644 --- a/docker/Alpine3.Dockerfile +++ b/docker/Alpine3.Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3-alpine3.15 +FROM python:3.12-alpine WORKDIR /tmp @@ -34,7 +34,7 @@ RUN apk add --no-cache bash>=5.1.16 # Setup system and install packages COPY scripts/os/setup_alpine3.sh /tmp/ -COPY requirements.txt /tmp/ +COPY pyproject.toml /tmp/ RUN cd /tmp && bash setup_alpine3.sh && rm -rf ./* # Install CI/CD scripts and linters config files diff --git a/docker/CentOS7.Dockerfile b/docker/CentOS7.Dockerfile index 80b70b039d4d72beadeda73aaf7ac44dbb186fc2..941ab554a9ea45aeeb607da4586e0cf5c9f07226 100644 --- a/docker/CentOS7.Dockerfile +++ b/docker/CentOS7.Dockerfile @@ -42,7 +42,7 @@ FROM centos:7 # Setup system and install packages COPY scripts/os/setup_centos7.sh /tmp/ -COPY requirements.txt /tmp/ +COPY pyproject.toml /tmp/ RUN cd /tmp && bash setup_centos7.sh && rm -rf ./* # Enable the SCL for all bash scripts. @@ -66,9 +66,3 @@ RUN cd /tmp && bash fetch.sh && rm -rf ./* # Remove all temporary files RUN rm -rf /tmp/* - -# Install CI/CD scripts and linters config files - -COPY scripts/os/install_files.sh /tmp/ -COPY .* scripts/cicd/*.* scripts/cicd/*/* /tmp/ -RUN bash /tmp/install_files.sh && rm -rf /tmp/* diff --git a/docker/CentOS8.Dockerfile b/docker/CentOS8.Dockerfile index ddd827a51f3eefc3558745d827b0046fd42a797b..1682fb3ce82e536c5bec05006c0bdd702ea1c6f0 100644 --- a/docker/CentOS8.Dockerfile +++ b/docker/CentOS8.Dockerfile @@ -19,7 +19,7 @@ WORKDIR /tmp # Setup system and install packages COPY scripts/os/setup_centos8.sh /tmp/ -COPY requirements.txt /tmp/ +COPY pyproject.toml /tmp/ RUN cd /tmp && bash setup_centos8.sh && rm -rf ./* # Enable the SCL for all bash scripts. @@ -43,9 +43,3 @@ RUN cd /tmp && bash fetch.sh && rm -rf ./* # Remove all temporary files RUN rm -rf /tmp/* - -# Install CI/CD scripts and linters config files - -COPY scripts/os/install_files.sh /tmp/ -COPY .* scripts/cicd/*.* scripts/cicd/*/* /tmp/ -RUN bash /tmp/install_files.sh && rm -rf /tmp/* diff --git a/docker/ClangFormat.Dockerfile b/docker/ClangFormat.Dockerfile index 4a05358bbbd4d942d269e097dd49d1cc6e7abead..87beaaa720a3b558f15bd31d2b2ec58c37340b01 100644 --- a/docker/ClangFormat.Dockerfile +++ b/docker/ClangFormat.Dockerfile @@ -20,11 +20,11 @@ # Build clang-format # #------------------------------------------------------------------------------# -FROM alpine:3.15 as clang-format-build +FROM alpine:3.18 as clang-format-build # Pass `--build-arg LLVM_TAG=master` for latest llvm commit ARG LLVM_TAG -ENV LLVM_TAG ${LLVM_TAG:-llvmorg-15-init} +ENV LLVM_TAG ${LLVM_TAG:-llvmorg-18-init} # HadoLint: It's ok to ignore this issue, as we don't care much about pkg versions. # hadolint ignore=DL3018 @@ -55,7 +55,7 @@ RUN cmake -GNinja \ # Install clang-format # #------------------------------------------------------------------------------# -FROM python:3-alpine3.15 +FROM python:3-alpine3.18 COPY --from=clang-format-build /build/llvm-project/build/bin/clang-format /usr/bin COPY --from=clang-format-build /build/llvm-project/clang/tools/clang-format/clang-format-diff.py /usr/local/bin/ diff --git a/docker/Conan.Dockerfile b/docker/Conan.Dockerfile index d620e302e167b0fcd073d1c940c191a8c7a480ad..1e1cd411bbf49db7997a10f3bf3679008da7b89f 100644 --- a/docker/Conan.Dockerfile +++ b/docker/Conan.Dockerfile @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 +FROM ubuntu:23.04 WORKDIR /tmp +# Define a gloval virtial environment for python packages. +# Actual dancing will be done in a setup script. + +ENV DOCKER_PYTHON_VENV=/opt/venv +ENV PATH="$DOCKER_PYTHON_VENV/bin:$PATH" + # Setup system and install packages COPY scripts/os/setup_conan.sh /tmp/ -COPY requirements.txt /tmp/ +COPY pyproject.toml /tmp/ RUN cd /tmp && bash setup_conan.sh && rm -rf ./* diff --git a/docker/HadoLint.Dockerfile b/docker/HadoLint.Dockerfile index ff30b3e3f7e6604f915026a991995d4901462963..2fba9c1ab96e89d85c34db600f70fec4f3b178bd 100644 --- a/docker/HadoLint.Dockerfile +++ b/docker/HadoLint.Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM hadolint/hadolint:v2.9.3-alpine +FROM hadolint/hadolint:v2.12.0-alpine # Setup system and install packages diff --git a/docker/Ubuntu2004.Dockerfile b/docker/Ubuntu2004.Dockerfile index 17582dcee9fbad8f64de48d41ac4a7ebbed38843..d78819070efc0abfd7de4d150e4adb63cbce6ad6 100644 --- a/docker/Ubuntu2004.Dockerfile +++ b/docker/Ubuntu2004.Dockerfile @@ -19,7 +19,7 @@ WORKDIR /tmp # Setup system and install packages COPY scripts/os/setup_ubuntu2004.sh /tmp/ -COPY requirements.txt /tmp/ +COPY pyproject.toml /tmp/ RUN cd /tmp && bash setup_ubuntu2004.sh && rm -rf ./* # Setup Conan @@ -40,11 +40,3 @@ RUN cd /tmp && bash fetch.sh && rm -rf ./* # Remove all temporary files RUN rm -rf /tmp/* - -# Install CI/CD scripts and linters config files - -COPY scripts/os/install_files.sh /tmp/ -COPY .* scripts/cicd/*.* scripts/cicd/*/* /tmp/ -RUN bash /tmp/install_files.sh && rm -rf /tmp/* - -ENV BASH_ENV "/etc/profile" diff --git a/docker/Ubuntu2304.Dockerfile b/docker/Ubuntu2304.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..996ec5e0aa6ee0e312a7a4f610acb7dfc3c65b82 --- /dev/null +++ b/docker/Ubuntu2304.Dockerfile @@ -0,0 +1,56 @@ +# Copyright 2020-2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:23.04 + +WORKDIR /tmp + +# Define a gloval virtial environment for python packages. +# Actual dancing will be done in a setup script. + +ENV DOCKER_PYTHON_VENV=/opt/venv +ENV PATH="$DOCKER_PYTHON_VENV/bin:$PATH" + +# Setup system and install packages + +COPY scripts/os/setup_ubuntu2304.sh /tmp/ +COPY pyproject.toml /tmp/ +RUN cd /tmp && bash setup_ubuntu2304.sh && rm -rf ./* + +# Setup Conan + +COPY conan/profiles/linux_clang \ + conan/profiles/linux_gcc \ + scripts/conan/setup.sh \ + /tmp/ +RUN cd /tmp && bash setup.sh && rm -rf ./* + +# Fetch Conan dependencies + +COPY conan/conanfile.txt \ + scripts/conan/fetch.sh \ + /tmp/ +RUN cd /tmp && bash fetch.sh && rm -rf ./* + +# Remove all temporary files + +RUN rm -rf /tmp/* + +# Install CI/CD scripts and linters config files + +COPY scripts/os/install_files.sh /tmp/ +COPY .* scripts/cicd/*.* scripts/cicd/*/* /tmp/ +RUN bash /tmp/install_files.sh && rm -rf /tmp/* + +ENV BASH_ENV "/etc/profile" diff --git a/docs/.dockerignore b/docs/.dockerignore deleted file mode 100644 index 58821e3e06cab8c2d62f487ca6b1cfa699ab59b4..0000000000000000000000000000000000000000 --- a/docs/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -!**.drawio -!**.png -!**.py.in diff --git a/docs/.gitignore b/docs/.gitignore index 58821e3e06cab8c2d62f487ca6b1cfa699ab59b4..bff8b61f1109f9eb473887bed186bc95f52fd4fd 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,3 @@ +!**.css !**.drawio !**.png -!**.py.in diff --git a/docs/_ext/gitlab_docs.py b/docs/_ext/gitlab_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..2fc44f64a1740fb18b8a265d11ab83762045afbf --- /dev/null +++ b/docs/_ext/gitlab_docs.py @@ -0,0 +1,77 @@ +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx.domains import Domain +from sphinx.roles import ReferenceRole +from sphinx.transforms import SphinxTransform + +if TYPE_CHECKING: + from sphinx.application import Sphinx + + +def _make_url(base_url: str, title: str = "", url_part: str = "") -> nodes.reference: + url = base_url + url_part + + if title: + reference = nodes.reference("", "", internal=False, refuri=url) + reference += nodes.strong(title, title) + else: + reference = nodes.reference("", url, internal=False, refuri=url) + + return reference + + +class BaseGitlabDocsRole(ReferenceRole): + base_url: str = "https://docs.gitlab.com/ee/" + section: str + + def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: + url = f"{self.base_url}/{self.section}/" + return [_make_url(url, self.title, self.target)], [] + + +class GitlabCiDocsRole(ReferenceRole): + section = "ci" + + +class GitlabUserDocsRole(ReferenceRole): + section = "user" + + +class GitlabDocsSubstitutions(SphinxTransform): + # run before the default Substitutions + default_priority = 210 + + def apply(self, **kwargs: dict) -> None: + base = _make_url("https://docs.gitlab.com") + ci = _make_url("https://docs.gitlab.com/ee/ci/") + user = _make_url("https://docs.gitlab.com/ee/user/") + + for ref in self.document.findall(nodes.substitution_reference): + refname = ref["refname"] + if refname == "gitlab-docs": + ref.replace_self(base) + elif refname == "gitlab-ci-docs": + ref.replace_self(ci) + elif refname == "gitlab-user-docs": + ref.replace_self(user) + + +class GitlabDocsDomain(Domain): + name = "gitlab-docs" + label = "gitLab.com official documentation" + roles = { + "ci": GitlabCiDocsRole(), + "user": GitlabUserDocsRole(), + } + + +def setup(app: "Sphinx"): + app.add_domain("gitlab-ci-docs", GitlabDocsDomain()) + app.add_transform(GitlabDocsSubstitutions) + + return { + "version": "0.0.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/_ext/gitlab_repo.py b/docs/_ext/gitlab_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..811b49a8e19b5101608fc2ee28f76b5557d5514c --- /dev/null +++ b/docs/_ext/gitlab_repo.py @@ -0,0 +1,127 @@ +from enum import StrEnum +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx.domains import Domain +from sphinx.errors import ExtensionError +from sphinx.roles import ReferenceRole, SphinxRole +from sphinx.transforms import SphinxTransform + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.config import Config + + +class ConfigValues(StrEnum): + BPROTO_REPO_LINK = "bproto_repo_link" + REPO_LINK = "repo_link" + + +def _make_url(title: str, url: str) -> nodes.reference: + if not title: + title = url + + return nodes.reference("", title, internal=False, refuri=url) + + +class BaseRepoRole(ReferenceRole): + config_name: str + url_part: str + + def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: + base_url: str = getattr(self.config, self.config_name) + refuri = f"{base_url}/-/{self.url_part}/{self.target}" + + return [_make_url(self.title, refuri)], [] + + +# bproto + + +class BProtoCommitRole(BaseRepoRole): + config_name = ConfigValues.BPROTO_REPO_LINK + url_part = "commit" + + +class BProtoIssueRole(BaseRepoRole): + config_name = ConfigValues.BPROTO_REPO_LINK + url_part = "issues" + + +class BProtoUrlRole(SphinxRole): + config_name = ConfigValues.BPROTO_REPO_LINK + + def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: + refuri = getattr(self.config, self.config_name) + return [_make_url(self.text, refuri)], [] + + +# In Shpinx we can define ":bproto-url:`title`". However ":bproto-url:" is an +# invalid use of Sphinx role. To overcome that we also define a "|bproto-url|" +# substitution, which might be quite handy when we just want to get a repo link. +class BProtoUrlSubstitutions(SphinxTransform): + config_name = ConfigValues.BPROTO_REPO_LINK + + # run before the default Substitutions + default_priority = 210 + + def apply(self, **kwargs: dict) -> None: + refuri = getattr(self.config, self.config_name) + node = _make_url("", refuri) + + for ref in self.document.findall(nodes.substitution_reference): + if ref["refname"] == "bproto-url": + ref.replace_self(node) + + +class BProtoDomain(Domain): + name = "bproto" + label = "bproto repo links" + roles = { + "commit": BProtoCommitRole(), + "issue": BProtoIssueRole(), + "url": BProtoUrlRole(), + } + + +# repo + + +class RepoFileRole(BaseRepoRole): + config_name = ConfigValues.REPO_LINK + url_part = "tree/main" + + +class RepoDomain(Domain): + name = "repo" + label = "gitlab repo links" + roles = { + "file": RepoFileRole(), + } + + +# setup + + +def _handle_inited_config(app: "Sphinx", config: "Config") -> None: + for config_name in ConfigValues: + if not getattr(config, config_name.value): + raise ExtensionError(f"{config_name.value!r} config value is empty!") + + +def setup(app: "Sphinx"): + for config_name in ConfigValues: + app.add_config_value(config_name.value, None, "env", [str]) + + app.connect("config-inited", _handle_inited_config) + + app.add_domain(BProtoDomain) + app.add_transform(BProtoUrlSubstitutions) + + app.add_domain(RepoDomain) + + return { + "version": "0.0.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/_images/artifacts.dark.png b/docs/_images/artifacts.dark.png new file mode 100644 index 0000000000000000000000000000000000000000..d0aca5f8040166025e6b94a20d83dcf96427fdaf Binary files /dev/null and b/docs/_images/artifacts.dark.png differ diff --git a/docs/_images/artifacts.drawio b/docs/_images/artifacts.drawio new file mode 100644 index 0000000000000000000000000000000000000000..aa11a92cc31b49dbda9d1534281848ef8b3ded4e --- /dev/null +++ b/docs/_images/artifacts.drawio @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/_images/artifacts.png b/docs/_images/artifacts.png new file mode 100644 index 0000000000000000000000000000000000000000..9675a20b84c86729b5d7456c00dfed2e36c24dc4 Binary files /dev/null and b/docs/_images/artifacts.png differ diff --git a/docs/_static/cicd/cpp_pipeline_flowchart.dark.png b/docs/_images/cpp_pipeline_flowchart.dark.png similarity index 100% rename from docs/_static/cicd/cpp_pipeline_flowchart.dark.png rename to docs/_images/cpp_pipeline_flowchart.dark.png diff --git a/docs/_static/cicd/cpp_pipeline_flowchart.drawio b/docs/_images/cpp_pipeline_flowchart.drawio similarity index 100% rename from docs/_static/cicd/cpp_pipeline_flowchart.drawio rename to docs/_images/cpp_pipeline_flowchart.drawio diff --git a/docs/_static/cicd/cpp_pipeline_flowchart.png b/docs/_images/cpp_pipeline_flowchart.png similarity index 100% rename from docs/_static/cicd/cpp_pipeline_flowchart.png rename to docs/_images/cpp_pipeline_flowchart.png diff --git a/docs/_static/cicd/docs_pipeline_flowchart.dark.png b/docs/_images/docs_pipeline_flowchart.dark.png similarity index 100% rename from docs/_static/cicd/docs_pipeline_flowchart.dark.png rename to docs/_images/docs_pipeline_flowchart.dark.png diff --git a/docs/_static/cicd/docs_pipeline_flowchart.drawio b/docs/_images/docs_pipeline_flowchart.drawio similarity index 100% rename from docs/_static/cicd/docs_pipeline_flowchart.drawio rename to docs/_images/docs_pipeline_flowchart.drawio diff --git a/docs/_static/cicd/docs_pipeline_flowchart.png b/docs/_images/docs_pipeline_flowchart.png similarity index 100% rename from docs/_static/cicd/docs_pipeline_flowchart.png rename to docs/_images/docs_pipeline_flowchart.png diff --git a/docs/_static/cicd/jobs_map.dark.png b/docs/_images/jobs_map.dark.png similarity index 100% rename from docs/_static/cicd/jobs_map.dark.png rename to docs/_images/jobs_map.dark.png diff --git a/docs/_static/cicd/jobs_map.drawio b/docs/_images/jobs_map.drawio similarity index 100% rename from docs/_static/cicd/jobs_map.drawio rename to docs/_images/jobs_map.drawio diff --git a/docs/_static/cicd/jobs_map.png b/docs/_images/jobs_map.png similarity index 100% rename from docs/_static/cicd/jobs_map.png rename to docs/_images/jobs_map.png diff --git a/docs/_static/cicd/linters_pipeline_flowchart.dark.png b/docs/_images/linters_pipeline_flowchart.dark.png similarity index 100% rename from docs/_static/cicd/linters_pipeline_flowchart.dark.png rename to docs/_images/linters_pipeline_flowchart.dark.png diff --git a/docs/_static/cicd/linters_pipeline_flowchart.drawio b/docs/_images/linters_pipeline_flowchart.drawio similarity index 100% rename from docs/_static/cicd/linters_pipeline_flowchart.drawio rename to docs/_images/linters_pipeline_flowchart.drawio diff --git a/docs/_static/cicd/linters_pipeline_flowchart.png b/docs/_images/linters_pipeline_flowchart.png similarity index 100% rename from docs/_static/cicd/linters_pipeline_flowchart.png rename to docs/_images/linters_pipeline_flowchart.png diff --git a/docs/_misc/.gitignore b/docs/_misc/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a1d35d9eb1141090381dd30fa9568e383d5b2069 --- /dev/null +++ b/docs/_misc/.gitignore @@ -0,0 +1 @@ +!*.* diff --git a/docs/_misc/cpp-project-dir-structure.txt b/docs/_misc/cpp-project-dir-structure.txt new file mode 100644 index 0000000000000000000000000000000000000000..85c91f7ce945a910256c2ec09fff84a13ea03af9 --- /dev/null +++ b/docs/_misc/cpp-project-dir-structure.txt @@ -0,0 +1,48 @@ +my-app + 3rd-party + 3rd-party-cpp-project-1 + ... + 3rd-party-cpp-project-2 + ... + CMakeLists.txt + docs + .gitignore + CMakeLists.txt + index.rst + project + module-1 + docs + api.rst + developer_notes.rst + index.rst + include + module-1 + module-1-api-header-1.hpp + ... + mocks + misc-project-1 + module-1-mock-header-1.hpp + ... + src + ... + tests + ... + CMakeLists.txt + module-2 + ... + misc + misc-module-1 + ... + mist-project-2 + ... + CMakeLists.txt + CMakeLists.txt + tools + lsan.supp + valgrind.supp + .gitignore + .gitlab-ci.yml + CMakeLists.txt + conanfile.py + README.md + requirements.txt diff --git a/docs/_static/css/s5defs.css b/docs/_static/css/s5defs.css new file mode 100644 index 0000000000000000000000000000000000000000..a5ea8e83e95b655b683fa7f757c6a852aea790c3 --- /dev/null +++ b/docs/_static/css/s5defs.css @@ -0,0 +1,93 @@ +/* + * CSS hacks and small modification for my Sphinx website + * :copyright: Copyright 2013-2016 Lilian Besson + * :license: GPLv3, see LICENSE for details. + */ + + +/* Colors and text decoration. + For example, :black:`text in black` or :blink:`text blinking` in rST. */ + +.black { + color: black; +} + +.gray { + color: gray; +} + +.grey { + color: gray; +} + +.silver { + color: silver; +} + +.white { + color: white; +} + +.maroon { + color: maroon; +} + +.red { + color: red; +} + +.magenta { + color: magenta; +} + +.fuchsia { + color: fuchsia; +} + +.pink { + color: pink; +} + +.orange { + color: orange; +} + +.yellow { + color: yellow; +} + +.lime { + color: lime; +} + +.green { + color: green; +} + +.olive { + color: olive; +} + +.teal { + color: teal; +} + +.cyan { + color: cyan; +} + +.aqua { + color: aqua; +} + +.blue { + color: blue; +} + +.navy { + color: navy; +} + +.purple { + color: purple; +} diff --git a/docs/conf.py b/docs/conf.py index 7ec192270dfb86c8ae7340e0ec38f6d45dde70df..ff0f3a77589ff9dfd9cab040fdb5c5c38da677c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,11 +6,16 @@ import datetime import os +import sys + +# -- Path setup -------------------------------------------------------------- + +sys.path.append(os.path.abspath("./_ext")) # -- Project information ----------------------------------------------------- author = "Zinchenko Serhii" -project = "Cpp Template Project" +project = "GitLab 3CT" copyright = f"2020-{datetime.datetime.now().year}, Zinchenko Serhii" # noqa: A001 @@ -18,6 +23,16 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +extensions = ["sphinx_copybutton", "gitlab_docs", "gitlab_repo"] + +# More can be found here: https://stackoverflow.com/a/60991308. +rst_prolog = """ +.. include:: + +.. By default below line is set to 'incremental' which breaks site a little. +.. default-role:: +""" + # -- Options for HTML output ------------------------------------------------- @@ -35,6 +50,9 @@ def _generate_last_updated_fmt(): return f"{commit_sha} ({branch}), %y-%m-%d %H:%M:%S" +html_css_files = ["css/s5defs.css"] +html_static_path = ["_static"] + html_copy_source = False html_last_updated_fmt = _generate_last_updated_fmt() @@ -73,3 +91,8 @@ def _generate_last_updated_fmt(): html_theme_options = {} html_title = project + +# -- Extensions ------------------------------------------------- + +bproto_repo_link = "https://gitlab.com/bproto/cicd" +repo_link = os.getenv("CI_PROJECT_URL") diff --git a/docs/dockers.rst b/docs/dockers.rst deleted file mode 100644 index 069e1daf892946bf2f114ee55b4d94df8156e0fb..0000000000000000000000000000000000000000 --- a/docs/dockers.rst +++ /dev/null @@ -1,334 +0,0 @@ -============= -Docker images -============= - -This page provides an information about all custom docker images, which are developed and maintained within this project. - -General purpose -=============== - -Here is listed all docker images with a very general purpose setup, that can be used in wide range of use-cases. - -Alpine 3 Docker -^^^^^^^^^^^^^^^ - -A utility image which contains all possible linters and ``sphinx`` related packages, therefore widely used in documentation and linters jobs. - -*TAGS* - -- ``alpine3:latest`` - -*BASE IMAGE* - Based on official Alpine 3 image with Python 3 (``python:3-alpine3.15``). - -*BUILD TIME* - 2 minutes on average. - -*SYSTEM PACKAGES* - -+-----------------+---------+-------------------------------------------------------------+ -| Package | Version | Purpose | -+=================+=========+=============================================================+ -| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | -+-----------------+---------+-------------------------------------------------------------+ -| ``curl`` | \- | Mainly to upload packages to GitLab. | -+-----------------+---------+-------------------------------------------------------------+ -| ``gcc`` | \- | **Only to build ``licenseheaders`` linter.** | -+-----------------+---------+-------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+-----------------+---------+-------------------------------------------------------------+ -| ``less`` | \- | To update ``less`` and bring support for additional flags. | -+-----------------+---------+-------------------------------------------------------------+ -| ``musl-dev`` | \- | **Only to build ``licenseheaders`` linter.** | -+-----------------+---------+-------------------------------------------------------------+ -| ``shellcheck`` | \- | To provided a shell linter. | -+-----------------+---------+-------------------------------------------------------------+ -| ``shfmt`` | \- | To provided a shell linter. | -+-----------------+---------+-------------------------------------------------------------+ - -*C++ PACKAGES* - None. - -*PYTHON PACKAGES* - Mainly linters and ``sphinx`` related packages. - -CentOS 7 Docker -^^^^^^^^^^^^^^^ - -A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. - -.. warning:: Very painful image with a huge amount of hacks and workarounds. - -*TAGS* - -- ``centos7:latest`` - -*BASE IMAGE* - Based on official CentOS 7 image (``centos:7``). - -*BUILD TIME* - 30 minutes on average. - -*SYSTEM PACKAGES* - -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| Package | Version | Purpose | -+==================================+=========+================================================================================+ -| ``cmake3`` | 3.21.0 | To provide a C/C++ build system. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-gcc`` | 10.2.1 | To provide a C compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-gcc-c++`` | 10.2.1 | To provide a C++ compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-gcc-gfortran`` | 10.2.1 | \- | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-gdb`` | 9.2 | To provide a C/C++ CLI debugger. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-libasan-devel`` | 10.2.1 | To provide address sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-liblsan-devel`` | 10.2.1 | To provide leak sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-libtsan-devel`` | 10.2.1 | To provide thread sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-libubsan-devel`` | 10.2.1 | To provide UB sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``devtoolset-10-make`` | 4.2.1 | \- | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``doxygen`` | 1.8.5 | To provide a documentation generator for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``llvm-toolset-13.0-clang`` | 13.0.1 | To provide a C/C++ compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``llvm-toolset-13.0-llvm`` | 13.0.1 | **To fix backtraces from sanitizers** | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``rh-python38-python`` | 3.8 | To provide support of ``python 3`` language. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``rh-python38-python-devel`` | 3.8 | To provide *dev* files for ``pybind11`` C++ library. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``rh-python38-python-pip`` | \- | To provide a python packages manager. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ - -*C++ PACKAGES* - Only those which are listed in ``Conan.cmake`` file. - -*PYTHON PACKAGES* - Mainly ``conan`` package and ``sphinx`` related packages. - -*CI SCRIPTS* - All CI scripts are included in this docker image. - -CentOS 8 Docker -^^^^^^^^^^^^^^^ - -A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. - -*TAGS* - -- ``centos8:latest`` - -*BASE IMAGE* - Based on official CentOS 8 image (``centos:8``). - -*BUILD TIME* - 30 minutes on average. - -*SYSTEM PACKAGES* - -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| Package | Version | Purpose | -+==================================+=========+================================================================================+ -| ``clang`` | 11.0.0 | To provide a C/C++ compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``cmake`` | 3.20.2 | To provide a C/C++ build system. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-gcc`` | 10.2.1 | To provide a C compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-gcc-c++`` | 10.2.1 | To provide a C++ compiler. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-gcc-gfortran`` | 10.2.1 | \- | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-gdb`` | 8.3 | To provide a C/C++ CLI debugger. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-libasan-devel`` | 10.2.1 | To provide address sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-liblsan-devel`` | 10.2.1 | To provide leak sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-libtsan-devel`` | 10.2.1 | To provide thread sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-toolset-10-libubsan-devel``| 10.2.1 | To provide UB sanitizer for code which uses C/C++ code. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``llvm`` | 12.0.1 | **To fix backtraces from sanitizers** | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``make`` | 4.2.1 | \- | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``python38-pip`` | 3.8 | To provide support of ``python 3`` language and bring a packages manager | -+----------------------------------+---------+--------------------------------------------------------------------------------+ -| ``python38-devel`` | \- | To provide *dev* files for ``pybind11`` C++ library. | -+----------------------------------+---------+--------------------------------------------------------------------------------+ - -*C++ PACKAGES* - Only those which are listed in ``Conan.cmake`` file. - -*PYTHON PACKAGES* - Mainly ``conan`` package and ``sphinx`` related packages. - -*CI SCRIPTS* - All CI scripts are included in this docker image. - -Ubuntu 20.04 Docker -^^^^^^^^^^^^^^^^^^^ - -A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. - -.. note:: As this system is currently a main one, it can be used as a reference to setup your own C/C++ workspace. - -*TAGS* - -- ``ubuntu2004:latest`` - -*BASE IMAGE* - Based on official Ubuntu 20.04 image (``ubuntu:20.04``). - -*BUILD TIME* - 25 minutes on average. - -*SYSTEM PACKAGES* - - **Package versions dated by 29 March 2023.** - -+----------------------+---------+--------------------------------------------------------------------------------+ -| Package | Version | Purpose | -+======================+=========+================================================================================+ -| ``clang-12`` | 12.0.0 | To provide a C/C++ compiler. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``cmake`` | 3.21.0 | To provide a C/C++ build system. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``cppcheck`` | 1.90 | To provided a C/C++ linter. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``curl`` | 7.68.0 | Mainly to upload packages to GitLab. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``doxygen`` | 1.8.17 | To provide a documentation generator for code which uses C/C++ code. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``g++-10`` | 10.3.0 | To provide a C compiler. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``gcc-10`` | 10.3.0 | To provide a C++ compiler. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``graphviz`` | 2.43.0 | To build diagrams and graphs. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``jq`` | \- | To provide ``sed`` for JSON data. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``libgtk-3-dev`` | 3.X | To provide *dev* files for ``sciter`` C++ library. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``libpython3.8-dev`` | 3.8 | To provide *dev* files for ``pybind11`` C++ library. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``lcov`` | 1.14 | To provide graphical front-end for GCC's coverage testing tool ``gcov``. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``llvm-12`` | 12.0.0 | **To fix backtraces from sanitizers** | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``make`` | 4.2.1 | \- | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``pkg-config`` | 3.X | Transitive dependency for ``gtk/system`` recipe, required by ```sciter```. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``python3-pip`` | 20.0.2 | To provide support of ``python 3.8.10`` language and bring a packages manager | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``valgrind`` | 3.15.0 | To provide leak sanitizer for code which uses C/C++ code. | -+----------------------+---------+--------------------------------------------------------------------------------+ -| ``wget`` | 1.20.3 | Mainly to download different stuff. | -+----------------------+---------+--------------------------------------------------------------------------------+ - -*C++ PACKAGES* - Only those which are listed in ``Conan.cmake`` file. - -*PYTHON PACKAGES* - Mainly ``conan`` package and ``sphinx`` related packages. - -*CI SCRIPTS* - All CI scripts are included in this docker image. - -Dedicated -========= - -ClangFormat Docker -^^^^^^^^^^^^^^^^^^ - -A docker image with a patched ``clang-format`` to support custom C++ code style. - -*TAGS* - -- ``clang-format:latest`` - -*BASE IMAGE* - Based on ``alpine:3.15`` and ``python:3-alpine3.15`` images (multi-stage build). - -*BUILD TIME* - 20 minutes on average. - -*SYSTEM PACKAGES* - -*Final* docker image will contain only below additional packages. - -+------------------+---------+-------------------------------------------------------------+ -| Package | Version | Purpose | -+==================+=========+=============================================================+ -| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | -+------------------+---------+-------------------------------------------------------------+ -| ``bash`` | \- | To provide support ``bash`` scripting language. | -+------------------+---------+-------------------------------------------------------------+ -| ``clang-format`` | \- | To provided a C/C++ formatter. | -+------------------+---------+-------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+------------------+---------+-------------------------------------------------------------+ - -*C++ PACKAGES* - None. - -*PYTHON PACKAGES* - None. - -*CI SCRIPTS* - Only ``clang-format`` related CI scripts are included in this docker image. - -HadoLint Docker -^^^^^^^^^^^^^^^ - -A custom ``hadolint`` docker image changed entrypoint and a bunch of new packages. - -*TAGS* - -- ``hadolint:latest`` - -*BASE IMAGE* - Based on official ``hadolint`` image (``hadolint/hadolint:v2.9.3-alpine``). - -*BUILD TIME* - 1 minute on average. - -*SYSTEM PACKAGES* - -*Final* docker image will contain only below additional packages. - -+------------------+---------+-------------------------------------------------------------+ -| Package | Version | Purpose | -+==================+=========+=============================================================+ -| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | -+------------------+---------+-------------------------------------------------------------+ -| ``bash`` | \- | To provide support ``bash`` scripting language. | -+------------------+---------+-------------------------------------------------------------+ -| ``git`` | \- | To been able to download GitLab project in a job container. | -+------------------+---------+-------------------------------------------------------------+ -| ``hadolint`` | \- | To provided a docker file linter. | -+------------------+---------+-------------------------------------------------------------+ - -*C++ PACKAGES* - None. - -*PYTHON PACKAGES* - None. - -*CI SCRIPTS* - Only ``hadolint`` related CI scripts are included in this docker image. diff --git a/docs/fork.rst b/docs/fork.rst new file mode 100644 index 0000000000000000000000000000000000000000..82218221c9cb2f9bc1b939d79754fff7242c5cd3 --- /dev/null +++ b/docs/fork.rst @@ -0,0 +1,67 @@ +.. _forking-project: + +=============== +Forking project +=============== + +.. _caching: https://docs.gitlab.com/ee/ci/caching +.. _fork: https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html + +The general rule of thumb can be answering positive to one of below questions: + +- do I need a different toolchain than is provided by current :ref:`pre-build docker image `? +- do I need a different set of python package than what is provided by current :ref:`pre-build docker image ` and caching_ can't help you with that? +- do I need to tweak :ref:`C++ pipeline ` implementation? + +Also it's worth to mention that some pipelines, like :ref:`C++ ` one, have :ref:`though limitations ` that can be overcome by caching_ packages but you still might decide to `fork `__ this project and change some lines. + +.. note:: + + Cookbook for GitLab forks can be found in `official documentation `__. + There you can find how to fork a project and how to maintain it. + +TODO list +========= + +Mandatory actions +----------------- + +#. Create a separate git branch. + .. note:: + + It's worth to mention - this is a rather personal choice. + However, if you are not planning to integrate your changes into :bproto:url:`parent project`, it's better to have a separate branch which contains only your changes. + +#. Change project path in :repo:file:`CI external files <.gitlab/ci/external>` from ``bproto/cicd`` to your own. + +#. Change macro prefix :repo:file:`clang-tidy config <.clang-tidy>` file from ``BPROTO`` to your own. + +#. Change include guards prefix from ``BPROTO`` to your own in an appropriate :repo:file:`shell script `. + +#. Add project specific python packages to :repo:file:`pyproject.toml `. + +#. Update :repo:file:`conan profiles ` accordingly to your needs. + .. note:: Current profiles set ``c++17`` standard. + +#. Add project specific conan packages to :repo:file:`conanfile.txt `. + +Optional actions +---------------- + +#. Update ``clang-format`` :repo:file:`config file <.clang-format>` to your needs. + +#. Update ``clang-tidy`` :repo:file:`config file <.clang-tidy>` to your needs. + +Automation +========== + +There is a simple shell script that can automate some of mentioned actions. +You can find it under :repo:file:`scripts/fork.sh` path. + +If you've forked :bproto:url:`bproto/cicd` into https://gitlab.com/supercoder/cicd, you can run script as follows: + +.. code-block:: bash + + bash scripts/fork.sh -p MY_PREFIX -r supercoder/cicd + +More information you can get by reading the script code, which is pretty straight forward. diff --git a/docs/git/tags.rst b/docs/git/tags.rst index 4b0e7865c7e39856d202a4a25102096da1348f72..0e946e05cc5fa5e9fd9f8b690467e7fcd5d7afff 100644 --- a/docs/git/tags.rst +++ b/docs/git/tags.rst @@ -90,7 +90,13 @@ That why it's hard to collect all possible cases when a particular tag should be +---------------------------+--------+----------------------------------------+ | Linux | Yes | | +---------------------------+--------+----------------------------------------+ - | Ubuntu 20.04 | Yes | | + | *Ubuntu 20.04* | **No** | | + + +--------+ + + | Ubuntu2004 | Yes | | + +---------------------------+--------+----------------------------------------+ + | *Ubuntu 23.04* | **No** | | + + +--------+ + + | Ubuntu2304 | Yes | | +---------------------------+--------+----------------------------------------+ | Windows | Yes | | +---------------------------+--------+----------------------------------------+ @@ -98,5 +104,7 @@ That why it's hard to collect all possible cases when a particular tag should be +---------------------------+--------+----------------------------------------+ | Conan | Yes | | +---------------------------+--------+----------------------------------------+ + | Hugo | Yes | | + +---------------------------+--------+----------------------------------------+ | Sphinx | Yes | | +---------------------------+--------+----------------------------------------+ diff --git a/docs/images/d/clang-format.rst b/docs/images/d/clang-format.rst new file mode 100644 index 0000000000000000000000000000000000000000..01ad1da8a1ec427e68eee3d1a9683ce39b546a1c --- /dev/null +++ b/docs/images/d/clang-format.rst @@ -0,0 +1,44 @@ +.. _images-dedicated-clangformat: + +============ +Clang-Format +============ + +A docker image with a patched ``clang-format`` to support a custom C++ code style. + +.. _alpine-image: https://hub.docker.com/_/alpine + +*TAGS* + +- ``clang-format:latest`` + +*BASE IMAGE* + Official `alpine:3.18 `_ and `python:3-alpine3.18 `_ images (multi-stage build). + +*BUILD TIME* + 20 minutes on average. + +*SYSTEM PACKAGES* + +*Final* docker image will contain only below additional packages. + ++------------------+---------+-------------------------------------------------------------+ +| Package | Version | Purpose | ++==================+=========+=============================================================+ +| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | ++------------------+---------+-------------------------------------------------------------+ +| ``bash`` | \- | To provide support ``bash`` scripting language. | ++------------------+---------+-------------------------------------------------------------+ +| ``clang-format`` | \- | To provided a C/C++ formatter. | ++------------------+---------+-------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++------------------+---------+-------------------------------------------------------------+ + +*C++ PACKAGES* + None. + +*PYTHON PACKAGES* + None. + +*CI SCRIPTS* + Only ``clang-format`` related CI scripts are included in this docker image. diff --git a/docs/images/d/conan.rst b/docs/images/d/conan.rst new file mode 100644 index 0000000000000000000000000000000000000000..1754d6a500a1f537eda23d2fb209d7bf55716bac --- /dev/null +++ b/docs/images/d/conan.rst @@ -0,0 +1,58 @@ +.. _images-dedicated-conan: + +===== +Conan +===== + +A custom ``conan`` docker image with a limited amount of packages. + +This docker image is provided as a clean setup for testing and deployment of ``conan`` packages that resides in `bPROTO recipes `__ group. + +*TAGS* + +- ``conan:latest`` + +*BASE IMAGE* + Official `ubuntu:23.04 `__ image. + +*BUILD TIME* + 1 minute on average. + +*SYSTEM PACKAGES* + +*Final* docker image will contain only below additional packages. + +.. caution:: Package versions dated by 17 October 2023. + ++------------------+---------+--------------------------------------------------------------------------------+ +| Package | Version | Purpose | ++==================+=========+================================================================================+ +| .. centered:: **Compilation** | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``g++-11`` | 11.4.0 | To provide a C compiler. | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-11`` | 11.4.0 | To provide a C++ compiler. | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``make`` | \- | \- | ++------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Essential** | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``python3-pip`` | 23.0.1 | To bring a python packages manager. | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``python3-venv`` | 3.11.2 | To overcome "externally managed environments" introduced by "PEP 668". | ++------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Utils** | ++------------------+---------+--------------------------------------------------------------------------------+ +| ``doxygen`` | \- | To provide a documentation generator for code which uses C/C++ code. | ++------------------+---------+--------------------------------------------------------------------------------+ + +*C++ PACKAGES* + None. + +*PYTHON PACKAGES* + Mainly ``conan`` package and ``sphinx`` related packages. + +*CI SCRIPTS* + No scripts are included. diff --git a/docs/images/d/hadolint.rst b/docs/images/d/hadolint.rst new file mode 100644 index 0000000000000000000000000000000000000000..9b2f2e145ab36b1f4e6d97985111ea82f9b3c567 --- /dev/null +++ b/docs/images/d/hadolint.rst @@ -0,0 +1,42 @@ +.. _images-dedicated-hadolint: + +======== +HadoLint +======== + +A custom ``hadolint`` docker image with changed entrypoint and a bunch of new packages. + +*TAGS* + +- ``hadolint:latest`` + +*BASE IMAGE* + Official `hadolint:v2.12.0-alpine `_ image. + +*BUILD TIME* + 1 minute on average. + +*SYSTEM PACKAGES* + +*Final* docker image will contain only below additional packages. + ++------------------+---------+-------------------------------------------------------------+ +| Package | Version | Purpose | ++==================+=========+=============================================================+ +| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | ++------------------+---------+-------------------------------------------------------------+ +| ``bash`` | \- | To provide support ``bash`` scripting language. | ++------------------+---------+-------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++------------------+---------+-------------------------------------------------------------+ +| ``hadolint`` | \- | To provided a docker file linter. | ++------------------+---------+-------------------------------------------------------------+ + +*C++ PACKAGES* + None. + +*PYTHON PACKAGES* + None. + +*CI SCRIPTS* + Only ``hadolint`` related CI scripts are included in this docker image. diff --git a/docs/images/d/index.rst b/docs/images/d/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..6f9f2e950b9da33633f8b8f8f5c259a6614c6422 --- /dev/null +++ b/docs/images/d/index.rst @@ -0,0 +1,17 @@ +================ +Dedicated images +================ + +Here is listed all docker images with a very limited setup, that were created to solve a very specific set of problems: + +.. toctree:: + :hidden: + :maxdepth: 1 + + clang-format.rst + conan.rst + hadolint.rst + +- :doc:`clang-format`, +- :doc:`conan`, +- :doc:`hadolint`. diff --git a/docs/images/g/alpine3.rst b/docs/images/g/alpine3.rst new file mode 100644 index 0000000000000000000000000000000000000000..742d5f867eed0cd87b5582392e63100f06f17d67 --- /dev/null +++ b/docs/images/g/alpine3.rst @@ -0,0 +1,55 @@ +.. _images-general-alpine3: + +======== +Alpine 3 +======== + +A utility image which contains all possible linters and ``sphinx`` related packages, therefore widely used in documentation and linters jobs. + +*TAGS* + +- ``alpine3:latest`` + +*BASE IMAGE* + Official `python:3.12-alpine `_ image. + +*BUILD TIME* + 2 minutes on average. + +*SYSTEM PACKAGES* + +.. caution:: Package versions dated by 29 March 2023. + ++-----------------+---------+-------------------------------------------------------------+ +| Package | Version | Purpose | ++=================+=========+=============================================================+ +| .. centered:: **Essential** | ++-----------------+---------+-------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++-----------------+---------+-------------------------------------------------------------+ +| .. centered:: **Utils** | ++-----------------+---------+-------------------------------------------------------------+ +| ``bat`` | \- | To provided syntax highlighting in GitLab terminal. | ++-----------------+---------+-------------------------------------------------------------+ +| ``curl`` | \- | Mainly to upload packages to GitLab. | ++-----------------+---------+-------------------------------------------------------------+ +| ``less`` | \- | To update ``less`` and bring support for additional flags. | ++-----------------+---------+-------------------------------------------------------------+ +| .. centered:: **Linters & Sanitizers** | ++-----------------+---------+-------------------------------------------------------------+ +| ``shellcheck`` | \- | To provided a shell scripts linter. | ++-----------------+---------+-------------------------------------------------------------+ +| ``shfmt`` | \- | To provided a shell scripts formatter. | ++-----------------+---------+-------------------------------------------------------------+ +| .. centered:: **Other** | ++-----------------+---------+-------------------------------------------------------------+ +| ``gcc`` | \- | **Only to build** ``licenseheaders`` **linter.** | ++-----------------+---------+-------------------------------------------------------------+ +| ``musl-dev`` | \- | **Only to build** ``licenseheaders`` **linter.** | ++-----------------+---------+-------------------------------------------------------------+ + +*C++ PACKAGES* + None. + +*PYTHON PACKAGES* + Mainly linters and ``sphinx`` related packages. diff --git a/docs/images/g/centos7.rst b/docs/images/g/centos7.rst new file mode 100644 index 0000000000000000000000000000000000000000..965146ab9041150f059d449d1779e26dca8546a4 --- /dev/null +++ b/docs/images/g/centos7.rst @@ -0,0 +1,62 @@ +.. _images-general-centos7: + +======== +CentOS 7 +======== + +A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. + +.. warning:: Very painful image with a huge amount of hacks and workarounds. + +*TAGS* + +- ``centos7:latest`` + +*BASE IMAGE* + Official `centos:7 `_ image. + +*BUILD TIME* + 30 minutes on average. + +*SYSTEM PACKAGES* + +.. caution:: Package versions dated by 29 March 2023. + ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| Package | Version | Purpose | ++==================================+=========+================================================================================+ +| .. centered:: **Compilation** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``cmake3`` | 3.21.0 | To provide a C/C++ build system. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``devtoolset-10-gcc`` | 10.2.1 | To provide a C compiler. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``devtoolset-10-gcc-c++`` | 10.2.1 | To provide a C++ compiler. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``devtoolset-10-gcc-gfortran`` | 10.2.1 | \- | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``devtoolset-10-gdb`` | 9.2 | To provide a C/C++ CLI debugger. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``devtoolset-10-make`` | 4.2.1 | \- | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Essential** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``rh-python38-python`` | 3.8 | To provide support of ``python 3`` language. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``rh-python38-python-pip`` | \- | To provide a python packages manager. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Other** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``rh-python38-python-devel`` | 3.8 | To provide *dev* files for ``pybind11`` C++ library. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ + +*C++ PACKAGES* + Only those which are listed in ``conan/conanfile.txt`` file. + +*PYTHON PACKAGES* + Only ``conan`` package. + +*CI SCRIPTS* + All CI scripts are included in this docker image. diff --git a/docs/images/g/centos8.rst b/docs/images/g/centos8.rst new file mode 100644 index 0000000000000000000000000000000000000000..5ad7e29d6e0c8d22bf9e23ca83421eb32a9d0770 --- /dev/null +++ b/docs/images/g/centos8.rst @@ -0,0 +1,62 @@ +.. _images-general-centos8: + +======== +CentOS 8 +======== + +.. warning:: + + Support of this image has been discontinued! + +A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. + +*TAGS* + +- ``centos8:latest`` + +*BASE IMAGE* + Official `centos:8 `_ image. + +*BUILD TIME* + 30 minutes on average. + +*SYSTEM PACKAGES* + +.. caution:: Package versions dated by 29 March 2023. + ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| Package | Version | Purpose | ++==================================+=========+================================================================================+ +| .. centered:: **Compilation** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``cmake`` | 3.20.2 | To provide a C/C++ build system. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-toolset-10-gcc`` | 10.2.1 | To provide a C compiler. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-toolset-10-gcc-c++`` | 10.2.1 | To provide a C++ compiler. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-toolset-10-gcc-gfortran`` | 10.2.1 | \- | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-toolset-10-gdb`` | 8.3 | To provide a C/C++ CLI debugger. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``make`` | 4.2.1 | \- | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Essential** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``python38-pip`` | 3.8 | To provide support of ``python 3`` language and bring a packages manager | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Other** | ++----------------------------------+---------+--------------------------------------------------------------------------------+ +| ``python38-devel`` | \- | To provide *dev* files for ``pybind11`` C++ library. | ++----------------------------------+---------+--------------------------------------------------------------------------------+ + +*C++ PACKAGES* + Only those which are listed in ``conan/conanfile.txt`` file. + +*PYTHON PACKAGES* + Only ``conan`` package. + +*CI SCRIPTS* + All CI scripts are included in this docker image. diff --git a/docs/images/g/index.rst b/docs/images/g/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a23e71833db7b1b2583addb7a1345c4e9f59040f --- /dev/null +++ b/docs/images/g/index.rst @@ -0,0 +1,21 @@ +====================== +General purpose images +====================== + +Here is listed all docker images with a very general purpose setup, that can be used in wide range of use-cases: + +.. toctree:: + :hidden: + :maxdepth: 1 + + alpine3.rst + centos7.rst + centos8.rst + ubuntu2004.rst + ubuntu2304.rst + +- :doc:`alpine3`, +- :doc:`centos7`, +- :doc:`centos8`, +- :doc:`ubuntu2004`, +- :doc:`ubuntu2304`. diff --git a/docs/images/g/ubuntu2004.rst b/docs/images/g/ubuntu2004.rst new file mode 100644 index 0000000000000000000000000000000000000000..356819775314811e296717ffbb1f5ac945dc0623 --- /dev/null +++ b/docs/images/g/ubuntu2004.rst @@ -0,0 +1,56 @@ +.. _images-general-ubuntu2004: + +============ +Ubuntu 20.04 +============ + +A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. + +.. note:: As this system is currently a main one, it can be used as a reference to setup your own C/C++ workspace. + +*TAGS* + +- ``ubuntu2004:latest`` + +*BASE IMAGE* + Official `ubuntu:20.04 `__ image. + +*BUILD TIME* + 25 minutes on average. + +*SYSTEM PACKAGES* + +.. caution:: Package versions dated by 29 March 2023. + ++----------------------+---------+--------------------------------------------------------------------------------+ +| Package | Version | Purpose | ++======================+=========+================================================================================+ +| .. centered:: **Compilation** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``cmake`` | 3.21.0 | To provide a C/C++ build system. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``g++-11`` | 10.3.0 | To provide a C compiler. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-11`` | 10.3.0 | To provide a C++ compiler. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``make`` | 4.2.1 | \- | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Essential** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``python3-pip`` | 20.0.2 | To provide support of ``python 3.8.10`` language and bring a packages manager | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Other** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``libpython3-dev`` | 3.8 | To provide *dev* files for ``pybind11`` C++ library. | ++----------------------+---------+--------------------------------------------------------------------------------+ + +*C++ PACKAGES* + Only those which are listed in ``conan/conanfile.txt`` file. + +*PYTHON PACKAGES* + Only ``conan`` package. + +*CI SCRIPTS* + All CI scripts are included in this docker image. diff --git a/docs/images/g/ubuntu2304.rst b/docs/images/g/ubuntu2304.rst new file mode 100644 index 0000000000000000000000000000000000000000..24fb33b109094dd07a8dd3c3fb92947de915aab4 --- /dev/null +++ b/docs/images/g/ubuntu2304.rst @@ -0,0 +1,80 @@ +.. _images-general-ubuntu2304: + +============ +Ubuntu 23.04 +============ + +A C++ based image which contains all required and optional packages to build C++ projects based on CMake build system. + +.. note:: As this system is currently a main one, it can be used as a reference to setup your own C/C++ workspace. + +*TAGS* + +- ``ubuntu2304:latest`` + +*BASE IMAGE* + Official `ubuntu:23.04 `__ image. + +*BUILD TIME* + 25 minutes on average. + +*SYSTEM PACKAGES* + +.. caution:: Package versions dated by 17 October 2023. + ++----------------------+---------+--------------------------------------------------------------------------------+ +| Package | Version | Purpose | ++======================+=========+================================================================================+ +| .. centered:: **Compilation** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``clang-15`` | 15.0.7 | To provide a C/C++ compiler. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``g++-11`` | 11.4.0 | To provide a C compiler. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``gcc-11`` | 11.4.0 | To provide a C++ compiler. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``cmake`` | 3.25.0 | To provide a C/C++ build system. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``llvm-15`` | 15.0.7 | **To fix backtraces from sanitizers** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``make`` | 4.3 | \- | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Essential** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``git`` | \- | To been able to download GitLab project in a job container. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``python3-pip`` | 23.0.1 | To bring a python packages manager. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``python3-venv`` | 3.11.2 | To overcome "externally managed environments" introduced by "PEP 668". | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Linters & Sanitizers** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``clang-tidy-15`` | 15.0.7 | Mainly to upload packages to GitLab. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``cppcheck`` | 2.10 | To provided a C/C++ linter. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``valgrind`` | 3.19.0 | To provide leak sanitizer for code which uses C/C++ code. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Utils** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``curl`` | 7.88.1 | Mainly to upload packages to GitLab. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``doxygen`` | 1.9.4 | To provide a documentation generator for code which uses C/C++ code. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``jq`` | \- | To provide ``sed`` for JSON data. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``wget`` | 1.21.3 | Mainly to download different stuff. | ++----------------------+---------+--------------------------------------------------------------------------------+ +| .. centered:: **Other** | ++----------------------+---------+--------------------------------------------------------------------------------+ +| ``libpython3-dev`` | 3.11.2 | To provide *dev* files for ``pybind11`` C++ library. | ++----------------------+---------+--------------------------------------------------------------------------------+ + +*C++ PACKAGES* + Only those which are listed in ``conan/conanfile.txt`` file. + +*PYTHON PACKAGES* + Mainly ``conan`` package and ``sphinx`` related packages. + +*CI SCRIPTS* + All CI scripts are included in this docker image. diff --git a/docs/images/index.rst b/docs/images/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..ca29c3ad6f7565bef78260aed5ca28e9918e2e26 --- /dev/null +++ b/docs/images/index.rst @@ -0,0 +1,40 @@ +.. _images: + +======================= +Pre-build docker images +======================= + +This section provides an information about all custom docker images, which are developed and maintained within this project. + +General purpose +=============== + +Here is listed all docker images with a very general purpose setup, that can be used in wide range of use-cases: + +.. toctree:: + :caption: General purpose + :hidden: + :maxdepth: 1 + + g/index.rst + +- :doc:`g/alpine3`, +- :doc:`g/centos7`, +- :doc:`g/centos8`, +- :doc:`g/ubuntu2004`. + +Dedicated +========= + +Here is listed all docker images with a very limited setup, that were created to solve a very specific set of problems: + +.. toctree:: + :caption: Dedicated + :hidden: + :maxdepth: 1 + + d/index.rst + +- :doc:`d/clang-format`, +- :doc:`d/conan`, +- :doc:`d/hadolint`. diff --git a/docs/index.rst b/docs/index.rst index aa6bcfc0f15a1a0cf46b2c19be05f0f762a327e0..7848a7d1c3b8042fd523f3c561773599b4725f90 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1,17 @@ -################## -Common CI/CD Tools -################## +========================= +GitLab Common CI/CD Tools +========================= .. toctree:: :caption: Documentation :maxdepth: 1 :hidden: + overview/index.rst + images/index.rst pipelines/index.rst - dockers.rst + fork.rst + variables.rst useful_links.rst .. toctree:: @@ -17,3 +20,81 @@ Common CI/CD Tools :hidden: git/index.rst + +.. _fork: https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html + +.. _`Zinchenko Serhii`: https://gitlab.com/b110011 + +**GitLab Common CI/CD Tools** (**GitLab 3CT** for short) provides a `GitLab CI/CD `__ solution that helps to control the quality of mainly C++ and Python codebases and their surrounding infrastructure. + +While the driving force of this module is to reduce duplication of CI/CD configurations and scripts across projects developed by me (`Zinchenko Serhii`_), it is intended to be useful for any software that wants to utilize GitLab CI/CD to some degree. + +This module also may serve as a useful reference for setting up a GitLab CI/CD using good practices. + +Usage +===== + +By design this project is intended to be used as a :gitlab-docs:ci:`CI/CD template ` in most of the cases. +However due to some limitations it might be `forked `__ to bring some changes and then to be used as your own CI/CD template. +Both ways will be covered in-deep down on this page. + +CI/CD templates +--------------- + +This project provides a bunch of useful :gitlab-docs:ci:`GitLab CI/CD pipelines ` for different programming languages and frameworks. +Below is a nice list of all available pipelines: + +- **Documentation** + - Hugo (:repo:file:`.gitlab/ci/external/hugo.yml`) + - Sphinx (:repo:file:`.gitlab/ci/external/sphinx.yml`) +- **Languages** + - C++ (:repo:file:`.gitlab/ci/external/cpp.yml`) + - CMake (:repo:file:`.gitlab/ci/external/lite.yml`) + - Docker (:repo:file:`.gitlab/ci/external/lite.yml`) + - Linux shell (:repo:file:`.gitlab/ci/external/lite.yml`) + - Python (:repo:file:`.gitlab/ci/external/lite.yml`) + - YAML (:repo:file:`.gitlab/ci/external/lite.yml`) +- **Packages** + - Conan (:repo:file:`.gitlab/ci/external/conan.yml`) + +You can start using those pipelines by including one of them as shown below: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' + +You also need to define below :gitlab-docs:ci:`CI/CD variables ` in your :gitlab-docs:ci:`group ` or :gitlab-docs:ci:`project ` CI/CD settings. + +.. raw:: html + + + ++----------+---------------------------+------------------------------------------+-----------+---------+--------------+ +| Type | Key | Value | Protected | Masked | Environments | ++==========+===========================+==========================================+===========+=========+==============+ +| Variable | BPROTO_CI_IMAGES_REGISTRY | ``registry.gitlab.com/bproto/artifacts`` | no | no | All | ++----------+---------------------------+------------------------------------------+-----------+---------+--------------+ + +Now you are ready to go! + +.. note:: + + To have a better understanding of pipelines implementation, you might also consider looking at an :ref:`overview ` of different aspects of pipelines, such as a list of implemented :ref:`stages ` and how the :ref:`workflow ` was designed. + +Forks +----- + +If you think, you need to fork this project and adjust it to your personal needs, please consider taking a look at :ref:`this documentation page `. + +License +======= + +All code is licensed under the `Apache 2.0 `__ license. diff --git a/docs/overview/artifacts.rst b/docs/overview/artifacts.rst new file mode 100644 index 0000000000000000000000000000000000000000..d697a3b72353b337a17c83946b838545ead67df0 --- /dev/null +++ b/docs/overview/artifacts.rst @@ -0,0 +1,35 @@ +.. _overview-artifacts: + +========= +Artifacts +========= + +.. _container-registry: https://docs.gitlab.com/ee/user/packages/container_registry/index.html +.. _package-registry: https://docs.gitlab.com/ee/user/packages/package_registry/index.html + +An artifacts system for this project is fully based on GitLab `package `_ and `container `_ registries and tries to utilize them to the fullest. + +It's still possible to use `Docker Hub `__ and `JFrog's Artifactory `__ but that will require some work to be done. + +.. + Flowchart has been generated with the help of https://app.diagrams.net/. + If you need to change flowchart: + 1. Go to the mentioned site. + 2. Import _static/artifacts.drawio. + 3. Change flowchart + 4. Save as .drawio and .png files. + 5. Update appropriate files in _static folder. + +.. image:: /_images/artifacts.png + :alt: Artifacts main repository + :class: only-light + +.. image:: /_images/artifacts.dark.png + :alt: Artifacts main repository + :class: only-dark + +Also despite the fact that any GitLab project can enable `package `_ and `container `_ registries, it has been decided to store all "shared" artifacts in a separate GitLab project called "artifacts". + +Here is an example of a such one: https://gitlab.com/bproto/artifacts. + +This approach helps to provide emulate a single source of artifacts and provide a simple way of distinguishing a "local" project only artifacts from those that are supposed to be shared across other projects too. diff --git a/docs/overview/index.rst b/docs/overview/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..56a4599a3e2ca22e4bde7f45fad3b3e47d13f3e0 --- /dev/null +++ b/docs/overview/index.rst @@ -0,0 +1,24 @@ +.. _overview: + +======== +Overview +======== + +This page is geared towards developers of the project CI/CD pipelines, and serves as a starting point of understanding core parts of pipelines. + +.. toctree:: + :maxdepth: 1 + :hidden: + + stages.rst + workflow.rst + artifacts.rst + +:doc:`stages` + An exhaustive overview of ideas behind pipeline stages. + +:doc:`workflow` + An overview of triggers for jobs from different stages. + +:doc:`artifacts` + An overview of project ideas around artifacts storage. diff --git a/docs/pipelines/overview.rst b/docs/overview/stages.rst similarity index 64% rename from docs/pipelines/overview.rst rename to docs/overview/stages.rst index b28d65adfde6dfea00be8d2fca1f2917db59d044..81dd593f21ac0b8d6ebb6d0cabe84f35142b2708 100644 --- a/docs/pipelines/overview.rst +++ b/docs/overview/stages.rst @@ -1,30 +1,6 @@ -======== -Overview -======== - -This page is geared towards developers of the project CI/CD pipelines, and serves as a starting point of understanding project pipelines. - -General workflow -================ - -A below table shows when jobs of a certain stage will be triggered. - -+--------+---------+-------------------+---------------+ -| Stage | Branch | CI_DEFAULT_BRANCH | Merge request | -+========+=========+===================+===============+ -| build | no | no | **yes** | -+--------+---------+-------------------+---------------+ -| test | no | no | **yes** | -+--------+---------+-------------------+---------------+ -| check | no | no | **yes** | -+--------+---------+-------------------+---------------+ -| report | no | no | **yes** | -+--------+---------+-------------------+---------------+ -| deploy | no | **yes** | no | -+--------+---------+-------------------+---------------+ -| misc | no | **yes** | no | -+--------+---------+-------------------+---------------+ +.. _overview-stages: +=============== Stages overview =============== @@ -38,44 +14,42 @@ Before you start reading a below overview of each pipeline stage, you should fam - Description * - Allow failure - - This section describes whether jobs are allowed to fail. + - This section describes whether job is allowed to fail. * - Artifacts - - This section describes what `job artifacts `__ will be generated after a successful run. + - This section describes what :gitlab-docs:ci:`job artifacts ` will be generated after a successful run. * - Cache - - This section describes used of `job's cache `__. + - This section describes used of :gitlab-docs:ci:`job's cache `. * - Dependencies - - This section describes what kind of dependencies tends to have. - Effectively combines `dependencies `__ and `needs `__ directives. + - This section describes what kind of dependencies job tends to have. + Effectively combines :gitlab-docs:ci:`dependencies ` and :gitlab-docs:ci:`needs ` directives. * - Environment - - This section describes what `environment variables `__ jobs use. + - This section describes what :gitlab-docs:ci:`environment variables ` jobs use. Right now only custom variables, those ones that defined using project settings, are taken into account. * - Images - - This section describes Docker images that the jobs should run in. - Effectively combines an overall content of `image `__ directive for all jobs. + - This section describes Docker images that the job is using. + Effectively combines an overall content of :gitlab-docs:ci:`image ` directive for all jobs. * - Triggers - - This section describes when at what cases jobs will be triggered. + - This section describes when and at what cases job will be triggered. -.. _overview-build-stage: +.. _overview-stages-build-stage: Build stage ----------- This is an *optional* stage for testing new changes that have been made to different parts of the project. -Especially this goes for C++ and Docker files, as these parts require assembling of mentions parts. +Especially this goes for C++ and `Docker `__ files, as these parts require assembling of mentions parts. *ALLOW FAILURE* Jobs are not allowed to fail as any failure means fundamental problems in one of the project parts. *ARTIFACTS* - Jobs generate C++ binaries for application executables and tests runners. - - Job artifacts also include all required CMake and CTest files for running tests. + Jobs may generate any kind of artifacts and might be used by jobs in other stages (like ``test`` stage). .. note:: Please take into account that generation of useful human-readable reports are usually delegated to jobs of the ``report`` stage. @@ -90,7 +64,7 @@ Especially this goes for C++ and Docker files, as these parts require assembling *TRIGGERS* Jobs are triggered only on **merge requests** and if a certain part of the projects have been modified (this can be C++ or Docker files). -.. _overview-test-stage: +.. _overview-stages-test-stage: Test stage ---------- @@ -101,7 +75,7 @@ This is an *optional* stage for running different types of project tests. Jobs are not allowed to fail as all project tests regardless their type must never be broken. *ARTIFACTS* - Jobs will generate `unit test reports `__. + Jobs will generate :gitlab-docs:ci:`unit test reports `. *DEPENDENCIES* Jobs require artifacts from parent jobs. @@ -113,7 +87,7 @@ This is an *optional* stage for running different types of project tests. *TRIGGERS* Jobs are triggered only on **merge requests** and only if parent job from the **build** stage hasn't failed. -.. _overview-check-stage: +.. _overview-stages-check-stage: Check stage ----------- @@ -137,7 +111,7 @@ This is an *optional* stage for executing different linters to diagnose code for *TRIGGERS* Jobs are triggered only on **merge requests** and if a certain part of the projects have been modified (this can be C++, Docker, Shell files and so on...). -.. _overview-report-stage: +.. _overview-stages-report-stage: Report stage ------------ @@ -161,7 +135,7 @@ This is an *optional* utility stage for generating reports for parent jobs in fr *TRIGGERS* Jobs are triggered only on **merge requests** and only if parent job has failed. -.. _overview-deploy-stage: +.. _overview-stages-deploy-stage: Deploy stage ------------ @@ -188,7 +162,7 @@ This is an *optional* stage for deploying different packages to a project or gro .. _registry: https://docs.gitlab.com/ee/user/packages/index.html -.. _overview-misc-stage: +.. _overview-stages-misc-stage: Misc stage ---------- diff --git a/docs/overview/workflow.rst b/docs/overview/workflow.rst new file mode 100644 index 0000000000000000000000000000000000000000..f22d450a39fb8ce673594ebfeaef04f7b51008a8 --- /dev/null +++ b/docs/overview/workflow.rst @@ -0,0 +1,68 @@ +.. _overview-workflow: + +======== +Workflow +======== + +Stages distribution +=================== + +A below table shows when jobs of a certain stage will be triggered. + +.. table:: + :width: 100% + + +--------+---------+-------------------+---------------+ + | Stage | Branch | CI_DEFAULT_BRANCH | Merge request | + +========+=========+===================+===============+ + | build | no | no | **yes** | + +--------+---------+-------------------+---------------+ + | test | no | no | **yes** | + +--------+---------+-------------------+---------------+ + | check | no | no | **yes** | + +--------+---------+-------------------+---------------+ + | report | no | no | **yes** | + +--------+---------+-------------------+---------------+ + | deploy | no | **yes** | no | + +--------+---------+-------------------+---------------+ + | misc | no | **yes** | no | + +--------+---------+-------------------+---------------+ + +Merge request labels +==================== + +.. _merge requests: https://docs.gitlab.com/ee/user/project/merge_requests/ +.. _label: https://docs.gitlab.com/ee/user/project/labels.html + +Current implementation provides a way to temporary or completely disable pipelines for specific `merge requests`_ by setting special label_. +Below is a list of such labels. + +.. note:: + Do please note that everything will work even you add only one label from the list. + +.. list-table:: + :header-rows: 1 + + * - Name + - Description + + * - ``ci::skip`` + - + + * - ``workflow::paused`` + - Merge requests / issues which are temporarily not worked on + +Also it's recommended, but not required, to add other useful labels. +However if you have already set-up your own labels you can safely skip this part. + +.. list-table:: + :header-rows: 1 + + * - Name + - Description + + * - ``workflow::blocked`` + - Merge requests / issues that are blocked until another issue has been completed + + * - ``workflow::in-development`` + - Merge requests / issues that are actively being worked on by a developer diff --git a/docs/pipelines/conan_pipeline.rst b/docs/pipelines/conan_pipeline.rst deleted file mode 100644 index 842e174d07363a1cb7bf1ca3492e34b62c82c670..0000000000000000000000000000000000000000 --- a/docs/pipelines/conan_pipeline.rst +++ /dev/null @@ -1,82 +0,0 @@ -============== -Conan Pipeline -============== - -This is a pipeline for building/testing and deploying new changes that have been made to anything that is directly related to the *conan recipe* part of the project. - -All jobs from docker pipeline stages are triggered only if changes have been made to *conan recipe* related files. -Right now such files are detected by these patterns: - -- ``conandata.yml``, -- ``conanfile.py``. - -**See also:** `Conan packages in the GitLab Package Registry `__. - -Overview -======== - -Build stage ------------ - -This stage is responsible for building conan packages from a clean state to make sure there are no unexpected results. - -*ALLOW FAILURE* - Jobs are not allowed to fail as any failure here means fundamental problems in a specific conan recipe. - -*ARTIFACTS* - Jobs don't generate any artifacts. - -*DEPENDENCIES* - Jobs don't require artifacts from other jobs. - - Jobs has no dependencies from other stages or jobs. - -*IMAGES* - Nothing specific here. - -*TRIGGERS* - Jobs are triggered only on **merge requests**. - -Deploy stage ------------- - -.. _package registry: https://docs.gitlab.com/ee/user/packages/package_registry - -This stage is responsible for deploying conan packages to the project `package registry`_. - -*ALLOW FAILURE* - Jobs are not allowed to fail as these jobs are run only on ``CI_DEFAULT_BRANCH`` after accepting a successful merge request! - -*ARTIFACTS* - New docker images will be deployed to the project `package registry`_. - -*DEPENDENCIES* - Jobs don't require artifacts from other jobs. - - Jobs has no dependencies from other stages or jobs. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_DEPLOY_USER - - Mandatory - - * - BPROTO_CI_DEPLOY_TOKEN - - Mandatory - - * - BPROTO_CI_CONAN_PACKAGE_NAME - - Mandatory - - * - BPROTO_CI_CONAN_PACKAGES_REGISTRY - - Mandatory - -*IMAGES* - Nothing specific here. - -*TRIGGERS* - Jobs are triggered only on ``CI_DEFAULT_BRANCH``. diff --git a/docs/pipelines/cpp_pipeline.rst b/docs/pipelines/cpp_pipeline.rst deleted file mode 100644 index 0619d4d95d0c461ba26b619b3a1195aee0549c21..0000000000000000000000000000000000000000 --- a/docs/pipelines/cpp_pipeline.rst +++ /dev/null @@ -1,153 +0,0 @@ -============ -C++ Pipeline -============ - -This is a pipeline for testing new changes that have been made to anything that is directly related to the C++ part of the project. -Here we try to build project from a clean state on different types of OS and with different set of build configurations to cover as much as possible. -Additionally to this we try to build project with all available sanitizers to detect different kind of issues as fast as possible. - -All jobs from C++ pipeline stages are triggered only on **merge requests** and only if changes have been made to C++ files. -Right now such files are detected by these patterns: - -- ``**/CMakeLists.txt``, -- ``**/*.cmake``, -- ``**/*.cpp``, -- ``**/*.hpp``, -- ``**/*.ipp``. - -.. note:: Please take into account that we don't include jobs from **check** stage, as they are not directly related to this pipeline. - -Flowchart -========= - -.. - Flowchart has been generated with the help of https://app.diagrams.net/. - If you need to change flowchart: - 1. Go to the mentioned site. - 2. Import _static/cicd/cpp_pipeline_flowchart.drawio. - 3. Change flowchart - 4. Save as .drawio and .png files. - 5. Update appropriate files in _static folder. - -.. image:: /_static/cicd/cpp_pipeline_flowchart.png - :target: ../_images/cicd/cpp_pipeline_flowchart.png - :alt: C++ Pipeline Flowchart - :class: only-light - -.. image:: /_static/cicd/cpp_pipeline_flowchart.dark.png - :target: ../_images/cicd/cpp_pipeline_flowchart.dark.png - :alt: C++ Pipeline Flowchart - :class: only-dark - -Stages overview -=============== - -Build stage ------------ - -*See also test stage overview from* :ref:`overview-build-stage`. - -*ALLOW FAILURE* - Jobs are not allowed to fail as any failure means project can't be fully assembled into distinct binary and library files. - -*ARTIFACTS* - Jobs generate C++ binaries for application executables and tests runners. - - Job artifacts also include all required CMake and CTest files for running tests. - -*DEPENDENCIES* - Jobs don't require artifacts from other jobs. - - Jobs have no dependencies from other stages or jobs. - -*IMAGES* - All jobs from this stage must use pre-build docker images to decrease overall pipeline time. - -Test stage ----------- - -This is a stage for running all available C++ tests. - -*See also test stage overview from* :ref:`overview-test-stage`. - -Report stage ------------- - -This is a stage to collect all available build information during the *build* stage and deploy it as a job artifacts if build has failed by any reason. - -*See also report stage overview from* :ref:`overview-report-stage`. - -Distribution -============ - -As we try to test as much configurations as possible, in this table all current setups will be tracked. - -.. note:: - Those options, that have been stress emphasized using *italic* and marked as (**yes**\*), are not taking part in a build process right now. - They are put here only to keep a track! - -+------------+-----------------------------------------------------------------------+ -| Options | System Configuration | -+ +-------------------+-------------------+---------------------+---------+ -| | CentOS 7 | CentOS 8 | Ubuntu 20.04 | Windows | -+ +---------+---------+---------+---------+---------+-----------+---------+ -| | Clang | GCC | Clang | GCC | Clang | GCC | MSBuild | -+============+=========+=========+=========+=========+=========+===========+=========+ -| | .. centered:: **Build Configuration** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Type | debug | release | debug | debug | debug | debug | debug | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Asserts | **yes** | no | **yes** | **yes** | **yes** | **yes** | **yes** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Cache | default | default | default | default | default | default | default | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Linter | default | default | default | default | default | default | default | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| LTO | no | **yes** | no | **yes** | no | **yes** | **yes** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| VCS Info | no | **yes** | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | .. centered:: **Binding** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Batch | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| JS | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Lua | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Python | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | .. centered:: **Docs** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | no | no | no | no | **yes** | **yes**\* | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | .. centered:: **Linters** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| *Coverage* | no | no | no | no | no | **yes**\* | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| *CppCheck* | no | no | no | no | no | **yes**\* | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| IWYU | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| LWYU | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | .. centered:: **Sanitizers** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| ASan | no | no | no | **yes** | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| LSan | no | no | no | **yes** | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| MSan | no | no | no | no | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| UBSan | no | no | no | **yes** | no | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| TSan | no | no | no | no | **yes** | no | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Valgrind | no | no | no | no | no | **yes** | no | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| | .. centered:: **Tests** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Benchmarks | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ -| Tests | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | -+------------+---------+---------+---------+---------+---------+-----------+---------+ diff --git a/docs/pipelines/docker_pipeline.rst b/docs/pipelines/docker_pipeline.rst deleted file mode 100644 index 9902871dd8bd79a14846a0e2430276ba7e209881..0000000000000000000000000000000000000000 --- a/docs/pipelines/docker_pipeline.rst +++ /dev/null @@ -1,80 +0,0 @@ -=============== -Docker Pipeline -=============== - -This is a pipeline for testing and deploying new changes that have been made to anything that is directly related to the *Docker* part of the project. - -All jobs from docker pipeline stages are triggered only if changes have been made to Docker related files. -Right now such files are detected by these patterns: - -- ``conan/*`` -- ``cmake/Conan.cmake``, -- ``docker/*.Dockerfile``, -- ``scripts/conan/`` -- ``scripts/os/setup_*.sh``, -- ``requirements.txt``. - -Overview -======== - -Build stage ------------ - -This stage is responsible for building Docker images from a clean state to make sure there are no unexpected results. - -*ALLOW FAILURE* - Jobs are not allowed to fail as any failure here means fundamental problems in a specific docker image. - -*ARTIFACTS* - Jobs don't generate any artifacts. - -*DEPENDENCIES* - Jobs don't require artifacts from other jobs. - - Jobs has no dependencies from other stages or jobs. - -*IMAGES* - Nothing specific here. - -*TRIGGERS* - Jobs are triggered only on **merge requests**. - -Deploy stage ------------- - -This stage is responsible for generated pre-build docker images for our pipelines. -So called pre-build images are crucial part of our project CI/CD as it allows us to decrease execution time of jobs. - -*ALLOW FAILURE* - Jobs are not allowed to fail as these jobs are run only on ``CI_DEFAULT_BRANCH`` after accepting a successful merge request! - -*ARTIFACTS* - New docker images will be deployed to the project `container registry `__. - -*DEPENDENCIES* - Jobs don't require artifacts from other jobs. - - Jobs has no dependencies from other stages or jobs. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_DEPLOY_USER - - Mandatory - - * - BPROTO_CI_DEPLOY_TOKEN - - Mandatory - - * - BPROTO_CI_IMAGES_REGISTRY - - Mandatory - -*IMAGES* - Nothing specific here. - -*TRIGGERS* - Jobs are triggered only on ``CI_DEFAULT_BRANCH``. diff --git a/docs/pipelines/docs/hugo.rst b/docs/pipelines/docs/hugo.rst new file mode 100644 index 0000000000000000000000000000000000000000..aca1d8edf12d13b1aa13fe3cb93b64f22d787d95 --- /dev/null +++ b/docs/pipelines/docs/hugo.rst @@ -0,0 +1,65 @@ +.. _pipelines-docs-hugo: + +==== +Hugo +==== + +This is a "sub-pipeline" for building and deploying new changes that have been made to `Hugo `__ files of a specific GitLab project. + +.. _pipelines-docs-hugo-triggers: + +Triggers +======== + +.. caution:: + + Right now all jobs from *Hugo "sub-pipeline"* stages are triggered on any change to repository. + +Jobs overview +============= + +build:docs +---------- + +This job is responsible for building HTML documentation from a clean state to make sure there are no unexpected results. + +*ALLOW FAILURE* + Job is not allowed to fail as any failure here means fundamental problems in documentation generation. + +*ARTIFACTS* + Job generate an artifact with a HTML documentation. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-build-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests**. + +pages +----- + +This stage is responsible for building and deploying static HTML documentation files to the project `GitLab Pages `__. + +*ALLOW FAILURE* + Job is not allowed to fail as any failure here means fundamental problems in documentation generation. + +*ARTIFACTS* + Job generate an artifact with a HTML documentation. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-deploy-stage`. + +*TRIGGERS* + Job is triggered only on ``CI_DEFAULT_BRANCH``. diff --git a/docs/pipelines/docs/index.rst b/docs/pipelines/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..304c14689b4edbd36aef2bad67a9d426a9b1a1bf --- /dev/null +++ b/docs/pipelines/docs/index.rst @@ -0,0 +1,15 @@ +==== +Docs +==== + +These documents cover in-detail all available "sub-pipelines" for documentation generation: + +.. toctree:: + :maxdepth: 1 + :hidden: + + hugo.rst + sphinx.rst + +- :doc:`hugo`, +- :doc:`sphinx`. diff --git a/docs/pipelines/docs/sphinx.rst b/docs/pipelines/docs/sphinx.rst new file mode 100644 index 0000000000000000000000000000000000000000..3d312d256f8139440773c33e64aaf325144d8ac1 --- /dev/null +++ b/docs/pipelines/docs/sphinx.rst @@ -0,0 +1,112 @@ +.. _pipelines-docs-sphinx: + +============= +Sphinx (reST) +============= + +.. _gitlab-pages: https://docs.gitlab.com/ee/user/project/pages + +This is a "sub-pipeline" for building and deploying new changes that have been made to `Sphinx `__ files of a specific GitLab project. + +.. note:: + + Current approach is way more simple than the old one and solely relies on `GitLab Pages `__. + If you still curious about how things worked before, you can look though :doc:`this page ` and investigate :bproto:commit:`this commit `. + +.. _pipelines-docs-sphinx-environment: + +Environment +=========== + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Default value + + * - :ref:`BPROTO_CI_SPHINX_SOURCE_DIR ` + - Optional + - ``docs`` + + * - :ref:`BPROTO_CI_SPHINX_WARNINGS_AS_ERRORS` + - Optional + - ``false`` + +.. _pipelines-docs-sphinx-triggers: + +Triggers +======== + +All jobs from *Sphinx "sub-pipeline"* stages are triggered and only if changes have been made to CMake files. +Right now such files are detected by these patterns: + +- ``docs/**/*`` +- ``project/**/CMakeLists.txt`` +- ``project/**/include/**/*.hpp`` + +As you can see from the above list of files pipeline will take into account both types of the project documentation: Doxygen (C++ files) and Sphinx (RST files) documentation. + +Jobs overview +============= + +.. _pipelines-docs-sphinx-jobs-overview-build-docs: + +build:docs +---------- + +This job is responsible for building HTML documentation from a clean state to make sure there are no unexpected results. + +*ALLOW FAILURE* + Job is not allowed to fail as any failure here means fundamental problems in documentation generation. + +*ARTIFACTS* + Job generate an artifact with a HTML documentation. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + A list of all environment variables can be found :ref:`here `. + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-build-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +report:docs +----------- + +This is a stage to simplify access to generated HTML files by :ref:`pipelines-docs-sphinx-jobs-overview-build-docs`. + +*See also report stage overview from* :ref:`overview-stages-report-stage`. + +pages +----- + +This stage is responsible for building and deploying static HTML documentation files to the project `GitLab Pages `_. + +*ALLOW FAILURE* + Job is not allowed to fail as any failure here means fundamental problems in documentation generation. + +*ARTIFACTS* + Job generate an artifact with a HTML documentation. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + A list of all environment variables can be found :ref:`here `. + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-deploy-stage`. + +*TRIGGERS* + Job is triggered only on ``CI_DEFAULT_BRANCH`` when any file from :ref:`trigger list ` is changed. diff --git a/docs/pipelines/docs_pipeline.rst b/docs/pipelines/docs/sphinx_old_approach.rst similarity index 91% rename from docs/pipelines/docs_pipeline.rst rename to docs/pipelines/docs/sphinx_old_approach.rst index c71a7bdd0258ec802f0feb271c5a5e0cde02d6cb..63812943de37bce0ef17bee310291f88b52a1bec 100644 --- a/docs/pipelines/docs_pipeline.rst +++ b/docs/pipelines/docs/sphinx_old_approach.rst @@ -1,6 +1,8 @@ -============= -Docs Pipeline -============= +:orphan: + +=============== +Docs generation +=============== This is a pipeline for testing and deploying new changes that have been made to anything that is directly related to the *documentation* part of the project. @@ -22,21 +24,19 @@ This is a very generalized flowchart that combines in itself all possible roots Flowchart has been generated with the help of https://app.diagrams.net/. If you need to change flowchart: 1. Go to the mentioned site. - 2. Import _static/cicd/docs_pipeline_flowchart.drawio. + 2. Import _static/docs_pipeline_flowchart.drawio. 3. Change flowchart 4. Save as .drawio and .png files. 5. Update appropriate files in _static folder. -.. image:: /_static/cicd/docs_pipeline_flowchart.png - :target: ../_images/cicd/docs_pipeline_flowchart.png +.. image:: /_images/docs_pipeline_flowchart.png :alt: Docs Pipeline Flowchart :class: only-light .. fill 4E220F, line EF8C61 fill 052853, line 4B9AFB -.. image:: /_static/cicd/docs_pipeline_flowchart.dark.png - :target: ../_images/cicd/docs_pipeline_flowchart.dark.png +.. image:: /_images/docs_pipeline_flowchart.dark.png :alt: Docs Pipeline Flowchart :class: only-dark diff --git a/docs/pipelines/index.rst b/docs/pipelines/index.rst index 21861deb077c64cda8ea3a779d89bf9753786145..2321aff7c52da62aa5954d90ed746037ea7c56d2 100644 --- a/docs/pipelines/index.rst +++ b/docs/pipelines/index.rst @@ -1,6 +1,6 @@ -######### -Pipelines -######### +============= +Sub-Pipelines +============= These documents cover in-detail all available "sub-pipelines" and their jobs. @@ -8,71 +8,50 @@ These documents cover in-detail all available "sub-pipelines" and their jobs. :maxdepth: 1 :hidden: - overview.rst - variables.rst - conan_pipeline.rst - cpp_pipeline.rst - docker_pipeline.rst - docs_pipeline.rst - linters_pipeline.rst - misc_pipeline.rst + docs/index.rst + lang/index.rst + misc.rst + packages/index.rst -Sub-Pipelines Concept -===================== +Concept +======= -We always try to compose everything into one top-level `GitLab CI/CD pipeline `__ that includes all jobs. -This way we can have all `GitLab CI/CD jobs `__, that have been triggered for merge request or a specific branch, in one place. +We always try to compose everything into one top-level :gitlab-docs:ci:`GitLab CI/CD pipeline ` that includes all jobs. +This way we can have all :gitlab-docs:ci:`GitLab CI/CD jobs `, that have been triggered for merge request or a specific branch, in one place. That said, we still try to decompose a top-level pipeline into smaller ones ("sub-pipelines"). Such approach decrease cognitive overload and makes it much easy to understand certain aspects of our pipeline. -Below table ("map") shows how jobs are distributed across different sub-pipelines. +Below table ("map") shows how jobs are distributed across different "sub-pipelines". .. Flowchart has been generated with the help of https://app.diagrams.net/. If you need to change flowchart: 1. Go to the mentioned site. - 2. Import _static/cicd/jobs_map.drawio. + 2. Import _static/jobs_map.drawio. 3. Change flowchart 4. Save as .drawio and .png files. 5. Update appropriate files in _static folder. -.. image:: /_static/cicd/jobs_map.png - :target: ../_images/cicd/jobs_map.png +.. image:: /_images/jobs_map.png :alt: Map of Sub-Pipelines Jobs :class: only-light -.. image:: /_static/cicd/jobs_map.dark.png - :target: ../_images/cicd/jobs_map.dark.png +.. image:: /_images/jobs_map.dark.png :alt: Map of Sub-Pipelines Jobs :class: only-dark -Sub-Pipelines List -================== +List +==== -:doc:`conan_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all CI jobs responsible for building and deploying conan packages. +:doc:`docs/index` + A collection "sub-pipeline"s for building and deploying documentation. -:doc:`cpp_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all CI jobs responsible for running C++ builds and tests. +:doc:`lang/index` + A collection of "sub-pipeline"s for building, testing and linting files related to different programming languages. -:doc:`docker_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all CI jobs responsible for running ``docker`` images builds and deployments. +:doc:`misc` + A "sub-pipeline" with different miscellaneous CI jobs. -:doc:`docs_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all CI jobs responsible for generating static HTML site with project documentation. - -:doc:`linters_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all CI jobs that uses linters. - -:doc:`misc_pipeline` - A GitLab CI/CD "sub-pipeline" which covers all misc CI jobs. - -Additional information -====================== - -:doc:`overview` - An exhaustive overview of "sub-pipelines" documentation with in-deep description of each section and part. - -:doc:`variables` - A list of all available environment variables with their description and use-cases. +:doc:`packages/index` + A collection "sub-pipeline"s for building and deploying different packages. diff --git a/docs/pipelines/lang/cmake.rst b/docs/pipelines/lang/cmake.rst new file mode 100644 index 0000000000000000000000000000000000000000..810f557256ca13123eb040377d64281df8ea8533 --- /dev/null +++ b/docs/pipelines/lang/cmake.rst @@ -0,0 +1,122 @@ +.. _pipelines-lang-cmake: + +===== +CMake +===== + +This is a "sub-pipeline" for linting new changes that have been made to CMake files of a specific GitLab project. + +.. _pipelines-lang-cmake-triggers: + +Triggers +======== + +All jobs from *CMake "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to CMake files. +Right now such files are detected by these patterns: + +- ``*/CMakeLists.txt``, +- ``**/*.cmake``. + +Jobs overview +============= + +check:cmake:format +------------------ + +This job executes `cmake-format `__ *only on changed CMake files* (not diff, but files) which will reformat code accordingly to the established project code style. + +.. note:: + This job is in active development phase and is a subject for a complete rework. + Please follow :bproto:issue:`this project issue <5>` to get more information about the current state of this job. + +*ALLOW FAILURE* + .. warning:: This job is allowed to fail as it's still in development. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.cmake-format.py`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:cmake:lint +---------------- + +This job executes `cmake-lint `__ *only on changed CMake files* (not diff, but files) which will diagnose and fix style violations like wrong variable and function names. + +.. note:: + This job is in active development phase and is a subject for a complete rework. + Please follow :bproto:issue:`this project issue <5>` to get more information about the current state of this job. + +*ALLOW FAILURE* + .. warning:: This job is allowed to fail as it's still in development. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.cmake-format.py`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Usage +===== + +In order to use above CI jobs, you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/lang/common.rst b/docs/pipelines/lang/common.rst new file mode 100644 index 0000000000000000000000000000000000000000..6c1ceabcd5b3a44b9baa5d3a1ffa94fe030c519b --- /dev/null +++ b/docs/pipelines/lang/common.rst @@ -0,0 +1,156 @@ +.. _pipelines-lang-common: + +====== +Common +====== + +This is a "sub-pipeline" for linting new changes that have been made to any kind of files of a specific GitLab project. + +Triggers +======== + +All jobs from *common "sub-pipeline"* stages are triggered only on merge requests if changes have been made to any file in git repository. + +Jobs overview +============= + +check:code:license +------------------ + +This job executes `licenseheaders `__ on project source code files, which will detect incorrect license headers. + +.. note:: + This job is in active development phase and is a subject for a complete rework. + Please follow :bproto:issue:`this project issue <9>` to get more information about the current state of this job. + +*ALLOW FAILURE* + .. warning:: This job is allowed to fail as it's still in development. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.license.tmpl`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job will be triggered only on **merge requests** and on all project files except ``3rd-party`` folder. + +check:code:spelling +------------------- + +This job executes `codespell `__ on project source code files, which will detect common misspellings. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.codespellrc`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job will be triggered only on **merge requests** and on all project files except ``3rd-party`` folder. + +check:commits:spelling +---------------------- + +This job performs various checks on commit messages for all found branch commits. + +Below is a list of implemented checks. + +- Check for common misspellings. +- Check that subject and body don't exceed 80 symbols limit. +- Check that GitHub issue references list follows ``GitHub: #1, #2, ...`` format. +- Check that GitHub issue references list follows ``GitLab: #1, #2, ...`` format. +- Check that JIRA issues list follows ``Jira: PROJ-1, PROJ-2, ...`` format. +- Check that tags list follows ``Tags: Bar, Foo, ...`` format. + +Here is an example of a valid message: + + | Fix foo to enable bar + | + | This fixes the broken behavior of the component by doing xyz. + | + | GitHub: #42, #43 + | GitLab: #315, #316 + | Jira: PROJ-101 + | Tags: Bar, Foo + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.codespellrc`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Origin + + * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - Mandatory + - GitLab + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job will be triggered only on **merge requests**. + +Usage +===== + +In order to use above CI jobs, you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/lang/cpp/build.rst b/docs/pipelines/lang/cpp/build.rst new file mode 100644 index 0000000000000000000000000000000000000000..330c23ecc99bc30383e1751da4f6c83bb1c39c1e --- /dev/null +++ b/docs/pipelines/lang/cpp/build.rst @@ -0,0 +1,173 @@ +============ +Build & Test +============ + +This document describes a part of C++ "sub-pipeline" which is responsible for building and testing new changes that have been made to C++ files of a specific GitLab project. + +Here we try to build project from a clean state on different OS's and with different set of build configurations to cover as much as possible. +Additionally to this we try to build project with all available sanitizers to detect different kind of issues as fast as possible. + +Flowchart +========= + +.. + Flowchart has been generated with the help of https://app.diagrams.net/. + If you need to change flowchart: + 1. Go to the mentioned site. + 2. Import _static/cpp_pipeline_flowchart.drawio. + 3. Change flowchart + 4. Save as .drawio and .png files. + 5. Update appropriate files in _static folder. + +.. image:: /_images/cpp_pipeline_flowchart.png + :alt: C++ Pipeline Flowchart + :class: only-light + +.. image:: /_images/cpp_pipeline_flowchart.dark.png + :alt: C++ Pipeline Flowchart + :class: only-dark + +Triggers +======== + +A list of triggers can be found :ref:`here `. + +Stages overview +=============== + +Build stage +----------- + +*ALLOW FAILURE* + Jobs are not allowed to fail as any failure means project can't be fully assembled into distinct binary and library files. + +*ARTIFACTS* + Jobs generate C++ binaries for application executables and tests runners. + + Job artifacts also include all required CMake and CTest files for running tests. + +*DEPENDENCIES* + Jobs don't require artifacts from other jobs. + + Jobs have no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Allowed values + - Default + + * - BPROTO_CI_CMAKE_ARGS + - Mandatory + - + - + + * - BPROTO_CI_CMAKE_BUILD_TYPE + - Optional + - | Debug, + | MinSizeRel, + | Release, + | RelWithDebInfo + - Debug + + * - BPROTO_CI_CMAKE_EXTRA_ARGS + - Optional + - + - + + * - CC + - Mandatory + - clang, gcc + - + + * - CXX + - Mandatory + - clang++, g++ + - + + * - CMAKE_GENERATOR + - Mandatory + - + - Unix Makefiles + +*IMAGES* + All jobs from this stage must use pre-build docker images to decrease overall pipeline time. + +*See also build stage overview from* :ref:`overview-stages-build-stage`. + +Test stage +---------- + +This is a stage for running all available C++ tests. + +*See also test stage overview from* :ref:`overview-stages-test-stage`. + +Report stage +------------ + +This is a stage to collect all available build information during the *build* stage and deploy it as a job artifacts if build has failed by any reason. + +*See also report stage overview from* :ref:`overview-stages-report-stage`. + +Distribution +============ + +As we try to test as much configurations as possible, in this table all current setups will be tracked. + ++-------------------+----------------------------------------------------------------------------+ +| Options | System Configuration | ++ +-------------+-------------+----------------+-------------------+-----------+ +| | CentOS 7 | *CentOS 8* | *Ubuntu 20.04* | Ubuntu 23.04 | *Windows* | ++ +-------------+-------------+----------------+-------------------+-----------+ +| | GCC | GCC | GCC | Clang | GCC | MSBuild | ++===================+=============+=============+================+=========+=========+===========+ +| | .. centered:: **Build Configuration** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Type | **release** | **release** | **release** | debug | debug | debug | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Asserts | no | no | no | **yes** | **yes** | **yes** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Cache | default | default | default | default | default | default | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Linter | default | default | default | default | default | default | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| LTO | **yes** | **yes** | **yes** | no | **yes** | **yes** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| | .. centered:: **Docs** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Docs | no | no | no | **yes** | no | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| | .. centered:: **Linters** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| LWYU | **yes** | **yes** | **yes** | **yes** | **yes** | **yes** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| | .. centered:: **Sanitizers** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| ASan | no | no | no | no | **yes** | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| LSan | no | no | no | no | **yes** | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| MSan | no | no | no | no | no | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| UBSan | no | no | no | no | **yes** | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| TSan | no | no | no | **yes** | no | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Valgrind | no | no | no | no | **yes** | no | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| | .. centered:: **Tests** | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ +| Benchmarks | .. centered:: **yes** | ++-------------------+ + +| Unit tests | | ++-------------------+ + +| Component tests | | ++-------------------+ + +| Integration tests | | ++-------------------+ + +| System tests | | ++-------------------+-------------+-------------+----------------+---------+---------+-----------+ diff --git a/docs/pipelines/lang/cpp/index.rst b/docs/pipelines/lang/cpp/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..39488642f9384ce759ac6ff999ef6f03b34004ec --- /dev/null +++ b/docs/pipelines/lang/cpp/index.rst @@ -0,0 +1,140 @@ +.. _pipelines-lang-cpp: + +=== +C++ +=== + +This is a "sub-pipeline" for building, testing and linting new changes that have been made to C++ files of a specific GitLab project. + +.. toctree:: + :maxdepth: 1 + :hidden: + + build.rst + linters.rst + +:doc:`build` + A document which covers all CI jobs responsible for building and testing C++ code. + +:doc:`linters` + A document which covers all CI jobs responsible for linting C++ code. + +.. _pipelines-lang-cpp-limitations: + +Limitations +=========== + +Unlike other "sub-pipeline" like :ref:`CMake `, :ref:`Python ` or :ref:`Shell ` ones, current implementation of C++ "sub-pipeline" is hard to use as a template because it has a few significant limitations. + +Core packages +^^^^^^^^^^^^^ + +Everything here is built using `CMake `__ build system and `Conan `__ package manager. + +It's still possible to integrate other build systems (like `Bazel `__, `b2 `__, `Meson `__ and `Premake `__) and package managers (like `CPM `__ and `vcpkg `__)! +However as of now I have no plans to do that because I use nothing from that and that will require a significant amount of work. + +CMake modules +^^^^^^^^^^^^^ + +Most jobs heavily relay on `CMOPTS `__ and `CppDoc `__ CMake modules. + +Those modules do all the heavy lifting like a seamless integration of sanitizes, linters and other useful tools, hence are essential for this "sub-pipeline". + +Pre-build Docker images +^^^^^^^^^^^^^^^^^^^^^^^ + +Due to the fact that all "sub-pipeline"s (this one is not an exception) try to utilizes :ref:`pre-build docker images ` as much as possible, this will be the rock on which we split in most of the cases. + +.. _gitlab-caching: https://docs.gitlab.com/ee/ci/caching +.. _gitlab-fork: https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html + +There is no easy way to add new `conan packages `__ to mentioned docker images so you need either to use `GitLab caching mechanism `_ to some degree or `fork `_ the project to handle that. + +Project structure +^^^^^^^^^^^^^^^^^ + +Most jobs, from the current implementation of the C++ "sub-pipeline", heavily relay on a certain C++ project structure that is shown below. + +.. Generated with the help of https://tree.nathanfriend.io. + +.. _"check:cpp:format": pipeline-lang-cpp-linters-format + +.. parsed-literal:: + + :olive:`green` - is an optional file or directory. + :red:`red` - is a mandatory file or directory. + + my-app/ + :olive:`├── 3rd-party/` :gray:`<- code here is ignored by linters.` + │ ├── 3rd-party-cpp-project-1/ + │ │ └── ... + │ ├── 3rd-party-cpp-project-2/ + │ │ └── ... + │ └── CMakeLists.txt + :olive:`├── docs/` :gray:`<- docs job looks for changes here.` + │ ├── .gitignore + │ ├── CMakeLists.txt + │ └── index.rst + :red:`├── project/` (:bproto:issue:`issue <7>`) :gray:`<- 'check:cpp:format' formats code only here.` + │ ├── module-1/ + │ │ :olive:`├── docs/` :gray:`<- docs job looks for changes here.` + │ │ │ ├── api.rst + │ │ │ ├── developer_notes.rst + │ │ │ └── index.rst + │ │ :olive:`├── include/` :gray:`<- docs job looks for changes here.` + │ │ │ └── module-1/ + │ │ │ ├── module-1-api-header-1.hpp + │ │ │ └── ... + │ │ ├── mocks/ + │ │ │ └── misc-project-1/ + │ │ │ ├── module-1-mock-header-1.hpp + │ │ │ └── ... + │ │ ├── src/ + │ │ │ └── ... + │ │ ├── tests/ + │ │ │ └── ... + │ │ └── CMakeLists.txt + │ ├── module-2/ + │ │ └── ... + │ :olive:`├── misc/` (:bproto:issue:`issue <6>`) :gray:`<- 'check:cpp:include_guards' handles this folder in a special way.` + │ │ ├── misc-module-1/ + │ │ │ └── ... + │ │ ├── mist-project-2/ + │ │ │ └── ... + │ │ └── CMakeLists.txt + │ └── CMakeLists.txt + :olive:`├── tools/` :gray:`<- sanitizers look for files with suppressions only here.` + │ ├── lsan.supp + │ └── valgrind.supp + ├── .gitignore + ├── .gitlab-ci.yml + :red:`├── CMakeLists.txt` :gray:`<- most jobs expect this file to be here.` + :olive:`├── conanfile.{py,txt}` :gray:`<- most jobs will use it if exists.` + ├── README.md + └── requirements.txt + +.. _pipelines-lang-cpp-linters-triggers: + +Triggers +======== + +Most of jobs from *C++ "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to C++ files. +Right now such files are detected by these patterns: + +- ``project/**/*.cpp``, +- ``project/**/*.hpp``, +- ``project/**/*.ipp``. + +Usage +===== + +In order to use this "sub-pipeline" you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/cpp.yml' diff --git a/docs/pipelines/lang/cpp/linters.rst b/docs/pipelines/lang/cpp/linters.rst new file mode 100644 index 0000000000000000000000000000000000000000..f9e7ab2ab09b556fbf63b2484b82e8f365985bd1 --- /dev/null +++ b/docs/pipelines/lang/cpp/linters.rst @@ -0,0 +1,456 @@ +======= +Linters +======= + +This document describes a part of C++ "sub-pipeline" which is responsible for linting new changes that have been made to C++ files of a specific GitLab project. + +Flowchart +========= + +This is a very generalized flowchart that combines in itself all possible roots of pipeline generation and execution. + +.. + Flowchart has been generated with the help of https://app.diagrams.net/. + If you need to change flowchart: + 1. Go to the mentioned site. + 2. Import _static/linters_pipeline_flowchart.drawio. + 3. Change flowchart + 4. Save as .drawio and .png files. + 5. Update appropriate files in _static folder. + +.. image:: /_images/linters_pipeline_flowchart.png + :alt: Linters Pipeline Flowchart + :class: only-light + +.. image:: /_images/linters_pipeline_flowchart.dark.png + :alt: Linters Pipeline Flowchart + :class: only-dark + +Triggers +======== + +Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Jobs overview +============= + +.. |manual-job| replace:: :gitlab-docs:ci:`manual one ` + +.. _linters-pipeline-check-cpp-coverage: + +check:cpp:coverage +------------------ + +This job calculates an overall coverage of C++ source code by tests. +In order to do that job builds project with compiler specific flags which allow to generate files that can be processed by `Gcov `__. + +At this moment job doesn't reuse artifacts from *build* or *test* stage. +Even so it means pipeline could take much less time, it still much easy to manage a job by itself. + +*ARTIFACTS* + Job will generate a set of files suitable for generating an HTML report. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_CMAKE_ARGS + - Mandatory + +*IMAGES* + :ref:`ubuntu2304:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + As this job is a |manual-job|, it can be triggered only by a user. + +report:cpp:coverage +------------------- + +*ARTIFACTS* + Job produces a statically generated HTML reports using `Gcovr `__. + +*DEPENDENCIES* + Jobs require artifacts from :ref:`linters-pipeline-check-cpp-coverage` job. + +*IMAGES* + `alpine:3 `__. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** and only if :ref:`linters-pipeline-check-cpp-coverage` job has failed. + +.. _pipeline-lang-cpp-linters-cppcheck: + +check:cpp:cppcheck +------------------ + +This job executes `cppcheck `__ *only on changed C++ files* (not diff, but files) which detect bugs and focuses on detecting undefined behavior and dangerous coding constructs. + +*ARTIFACTS* + Job will generate a set of files suitable for generating an HTML report. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.cppcheck`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_CMAKE_GLOBAL_ARGS + - Mandatory + +*IMAGES* + :ref:`ubuntu2304:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + As this job is a |manual-job|, it can be triggered only by a user. + +report:cpp:cppcheck +------------------- + +*ARTIFACTS* + Job produces a statically generated HTML reports using ``cppcheck-htmlreport`` from the `cppcheck `__ package. + +*DEPENDENCIES* + Jobs require artifacts from :ref:`pipeline-lang-cpp-linters-cppcheck` job. + +*IMAGES* + `alpine:3 `__. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** and only if :ref:`pipeline-lang-cpp-linters-cppcheck` job has failed. + +.. _pipeline-lang-cpp-linters-format: + +check:cpp:format +---------------- + +This job executes `clang-format `__ *only on changes from C++ files* which will reformat code accordingly to the established project code style. + +.. warning:: + This job uses a docker image with a patched ``clang-format`` to support a custom C++ code style. + +*ALLOW FAILURE* + .. warning:: This job is allowed to fail as it's still in development. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + ``clang-format`` for each input file will try to find the ``.clang-format`` file located in the closest parent directory of the input file. + Different configuration options can be found on `official page `__ + + Default config, which can be found :repo:file:`here <.clang-format>` and shipped with project C++ related docker images, implements the `following formatting conventions `__ and it's heavily opinionated. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Origin + + * - CI_COMMIT_SHA + - Mandatory + - GitLab + + * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - Mandatory + - GitLab + + * - BPROTO_CI_LINTER_TRACE + - Optional + - + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - + +*IMAGES* + :ref:`clang-format:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:cpp:include_guards +------------------------ + +This job executes ``checkguard`` from `guardonce `__ package *only on changed C++ files* (not diff, but files) which will detect wrong written `include guards `__. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Current config is hard-coded in :repo:file:`this file ` and has the following value: + + - ``path | remove - | remove _ | prepend __BPROTO_ | append __ | upper`` + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Origin + + * - CI_PROJECT_DIR + - Mandatory + - GitLab + + * - BPROTO_CI_LINTER_TRACE + - Optional + - + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:cpp:iostream +------------------ + +This job executes a simple shell script to find files that includes `iostream `__ library header and report them. + +**Preface** + +.. https://stackoverflow.com/questions/189562 + +.. epigraph:: + + "The most effective debugging tool is still careful thought, coupled with judiciously placed print statements." + + -- Brian Kernighan, "Unix for Beginners" (1979) + +``printf()`` debugging - useful and powerful technique. That said I often forget to remove redundant ```` header after debugging is done. + +It might sound as a small issue however including the ```` header can significantly impact compile time, as it is one of the heaviest standard C++ headers. + +**Suppressing warnings** + +If you need to silence the warning, you can put ``// ignore`` comment on the same line as an ``include`` directive. + +.. code-block:: cpp + + #include // ignore + +*ARTIFACTS* + Job doesn't generate any artifacts. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +.. _pipeline-lang-cpp-linters-tidy: + +check:cpp:tidy +-------------- + +This job executes `clang-tidy `__ *only on a diff* which will diagnose and fix typical programming errors, like style violations, interface misuse, or bugs that can be deduced via static analysis. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.clang-tidy`` name. + + Default config, which can be found :repo:file:`here <.clang-tidy>` and shipped with project C++ related docker images, implements the `following naming conventions `__ and is rather restrictive. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Origin + + * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - Mandatory + - GitLab + + * - BPROTO_CI_LINTER_TRACE + - Optional + - + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - + +*IMAGES* + :ref:`ubuntu2304:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + As this job is a |manual-job|, it can be triggered only by a user. + +check:cpp:tidy (nightly) +------------------------ + +This is almost the same job as :ref:`pipeline-lang-cpp-linters-tidy` with one slight difference. +This is a nightly job which checks entire code base on a daily bases. + +The main reason for such separation is ability to detect a set of issues that is impossible to find while only checking files diff. +The downside of this solution is a long running job, which can take more than half an hour to run. + +However that issue is mitigated a little bit by caching current job result and a commit hash on which that job has run. +So the next job run might do nothing if commit hash hasn't changed from the time of the last run. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + + Also it will generate and save a simple package (``nightly_clang_tidy``) that contains a result of current job run and a commit hash on which the job has run. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.clang-tidy`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`ubuntu2304:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +Unsupported linters +=================== + +Here is a list of C++ linters that haven't been integrated yet. + +**ClangBuildAnalyzer** + Link: https://github.com/aras-p/ClangBuildAnalyzer + + Clang build analysis tool using ``-ftime-trace``. + +**Clang-Include-Fixer** + Link: https://clang.llvm.org/extra/clang-include-fixer.html + + One of the major nuisances of C++ compared to other languages is the manual management of ``#include`` directives in any file. + ``clang-include-fixer`` addresses one aspect of this problem by providing an automated way of adding ``#include`` directives for missing symbols in one translation unit. + +**Clang Static Analyzer** + Link: https://clang-analyzer.llvm.org/ + + The Clang Static Analyzer is a source code analysis tool that finds bugs in C, C++, and Objective-C programs. + +**CodeChecker** + Link: https://github.com/Ericsson/codechecker + + CodeChecker is an analyzer tooling, defect database and viewer extension for the Clang Static Analyzer and Clang-Tidy. + +**flawfinder** + Link: https://github.com/david-a-wheeler/flawfinder + + A static analysis tool for finding vulnerabilities in C/C++ source code. + +**ikos** + Link: https://github.com/NASA-SW-VnV/ikos + + Static analyzer for C/C++ based on the theory of Abstract Interpretation. + +**Infer** + Links: https://github.com/facebook/infer + + A static analyzer for Java, C, C++, and Objective-C. + +**Klocwork** + Link: https://www.perforce.com/products/klocwork + + Klocwork static code analysis and SAST tool for C, C++, C#, Java, JavaScript, Python, and Kotlin identifies software security, quality, and reliability issues helping to enforce compliance with standards. + +**krazy** + Link: https://github.com/Krazy-collection/krazy + + An extensible framework for performing static code checking on files of any format. + +**OCLint** + Link: https://github.com/oclint/oclint + + A static source code analysis tool to improve quality and reduce defects for C, C++ and Objective-C. + +**stack** + Link: https://css.csail.mit.edu/stack + + An analyzer that tries to report code that might be removed by an optimizer. diff --git a/docs/pipelines/lang/docker/build.rst b/docs/pipelines/lang/docker/build.rst new file mode 100644 index 0000000000000000000000000000000000000000..9c9e777b28a3a3001b8fcef173d806639078f91b --- /dev/null +++ b/docs/pipelines/lang/docker/build.rst @@ -0,0 +1,159 @@ +============== +Build & Deploy +============== + +This is a pipeline for testing and deploying new changes that have been made to anything that is directly related to the *Docker* part of the project. + +.. _pipelines-lang-docker-environment: + +Environment +=========== + +.. list-table:: + :header-rows: 1 + :width: 100% + + * - Name + - Necessity + - Origin + + * - BPROTO_CI_DEPLOY_USER + - Mandatory + - + + * - BPROTO_CI_DEPLOY_TOKEN + - Mandatory + - + + * - BPROTO_CI_IMAGES_REGISTRY + - Mandatory + - + + * - *CI_COMMIT_REF_NAME* + - Mandatory + - GitLab + + * - *CI_DEFAULT_BRANCH* + - Mandatory + - GitLab + +.. _pipelines-lang-docker-build-triggers: + +Triggers +======== + +All jobs from *Docker "sub-pipeline"* stages are triggered and only if changes have been made to docker files. +Right now such files are detected by below patterns: + +- ``conan/*`` +- ``cmake/Conan.cmake``, +- ``docker/*.Dockerfile``, +- ``scripts/conan/`` +- ``scripts/os/setup_*.sh``, +- ``requirements.txt``. + +Stages overview +=============== + +Build stage +----------- + +This stage is responsible for building `Docker `__ images from a clean state to make sure there are no unexpected results. + +*ALLOW FAILURE* + Jobs are not allowed to fail as any failure means fundamental problems in a specific docker image. + +*ARTIFACTS* + Jobs don't generate any artifacts. + +*DEPENDENCIES* + Jobs don't require artifacts from other jobs. + + Jobs has no dependencies from other stages or jobs. + +*ENVIRONMENT* + A list of all environment variables can be found :ref:`here `. + +*IMAGES* + `docker:19.03.12 `__. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Deploy stage +------------ + +This stage is responsible for generating pre-build docker images for our pipelines. +So called pre-build images are crucial part of our CI/CD as it allows us to decrease execution time of jobs. + +*ALLOW FAILURE* + Jobs are not allowed to fail as these jobs are run only on ``CI_DEFAULT_BRANCH`` after accepting a successful merge request! + +*ARTIFACTS* + New docker images will be deployed to the project `container registry `__. + +*DEPENDENCIES* + Jobs don't require artifacts from other jobs. + + Jobs has no dependencies from other stages or jobs. + +*ENVIRONMENT* + A list of all environment variables can be found :ref:`here `. + +*IMAGES* + `docker:19.03.12 `__. + +*TRIGGERS* + Job is triggered only on ``CI_DEFAULT_BRANCH`` when any file from :ref:`trigger list ` is changed. + +Jobs overview +============= + +.. table:: + :width: 100% + :widths: 40 30 30 + + +--------------------------------+--------+----------------------------------------------------+ + | Job name | Stage | Image | + +================================+========+====================================================+ + | .. centered:: **General purpose images** | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:alpine3`` | build | :ref:`Alipne 3 ` | + +--------------------------------+--------+ + + | ``deploy:docker:alpine3`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:centos7`` | build | :ref:`CentOS 7 ` | + +--------------------------------+--------+ + + | ``deploy:docker:centos7`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:centos8`` | build | :ref:`CentOS 7 ` | + +--------------------------------+--------+ + + | ``deploy:docker:centos8`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:ubuntu2004`` | build | :ref:`Ubuntu 20.04 ` | + +--------------------------------+--------+ + + | ``deploy:docker:ubuntu2004`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:ubuntu2304`` | build | :ref:`Ubuntu 20.04 ` | + +--------------------------------+--------+ + + | ``deploy:docker:ubuntu2304`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | .. centered:: **Dedicated images** | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:clang-format`` | build | :ref:`Clang-Format ` | + +--------------------------------+--------+ + + | ``deploy:docker:clang-format`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:conan`` | build | :ref:`Conan ` | + +--------------------------------+--------+ + + | ``deploy:docker:conan`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + | ``build:docker:hadolint`` | build | :ref:`HadoLint ` | + +--------------------------------+--------+ + + | ``deploy:docker:hadolint`` | deploy | | + +--------------------------------+--------+----------------------------------------------------+ + +Usage +===== + +This is an internal part of the Docker "sub-pipeline" which meant to be used solely by :bproto:url:`3CT: Common CI/CD Tools` project. diff --git a/docs/pipelines/lang/docker/index.rst b/docs/pipelines/lang/docker/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..71d0a1e06667c3fa0318036dc2c4eebc135f2303 --- /dev/null +++ b/docs/pipelines/lang/docker/index.rst @@ -0,0 +1,20 @@ +.. _pipelines-lang-docker: + +====== +Docker +====== + +This is a "sub-pipeline" for building, linting and deploying new changes that have been made to Docker files of a specific GitLab project. + +.. toctree:: + :maxdepth: 1 + :hidden: + + build.rst + linters.rst + +:doc:`build` + A document which covers all CI jobs responsible for building and deploying docker images. + +:doc:`linters` + A document which covers all CI jobs responsible for linting docker files. diff --git a/docs/pipelines/lang/docker/linters.rst b/docs/pipelines/lang/docker/linters.rst new file mode 100644 index 0000000000000000000000000000000000000000..41b2dbde1e186c5dfd5c452674d8ad818e25b2f6 --- /dev/null +++ b/docs/pipelines/lang/docker/linters.rst @@ -0,0 +1,68 @@ +======= +Linters +======= + +This is a "sub-pipeline" for linting new changes that have been made to `docker `__ files of a specific GitLab project. + +.. _pipelines-lang-docker-linters-triggers: + +Triggers +======== + +All jobs from *Docker "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to docker files. +Right now such files are detected by these patterns: + +- ``**/.Dockerfile``. + +Jobs overview +============= + +check:docker:lint +----------------- + +This job executes `hadolint `__ *only on changed Docker files* (not diff, but files) which will diagnose bugs that can be deduced via static analysis. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.hadolint.yaml`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`hadolint:latest `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Usage +===== + +In order to use above CI jobs, you need to add the follow code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/lang/index.rst b/docs/pipelines/lang/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..c414fc0012ea485ef3994f4ac252b3ea76316cf3 --- /dev/null +++ b/docs/pipelines/lang/index.rst @@ -0,0 +1,25 @@ +========= +Languages +========= + +These documents cover in-detail all available "sub-pipelines" for different programming languages: + +.. toctree:: + :maxdepth: 1 + :hidden: + + common.rst + cmake.rst + cpp/index.rst + docker/index.rst + python/index.rst + shell.rst + yaml.rst + +- :doc:`common`, +- :doc:`cmake`, +- :doc:`cpp/index`, +- :doc:`docker/index`, +- :doc:`python/index`, +- :doc:`shell`, +- :doc:`yaml`. diff --git a/docs/pipelines/lang/python/build.rst b/docs/pipelines/lang/python/build.rst new file mode 100644 index 0000000000000000000000000000000000000000..1a29781370720280460ef29c586a86f5d4600854 --- /dev/null +++ b/docs/pipelines/lang/python/build.rst @@ -0,0 +1,55 @@ +============ +Build & Test +============ + +There is no CI jobs in current CI/CD implementation, but there is couple of code snippets that shows how easy it to create a CI job to build and test python code. + +Global cache +============ + +.. code-block:: yaml + + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + + cache: + key: + files: + - requirements.txt + paths: + - .cache/pip + + example: + image: python:3-alpine3.15 + stage: build-test + before_script: + - pip install -r requirements.txt + script: + # build a package + - pip install . + # or run tests + - pytest + +Job's cache +=========== + +.. code-block:: yaml + + example: + image: python:3-alpine3.15 + stage: build-test + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + before_script: + - pip install -r requirements.txt + script: + # build a package + - pip install . + # or run tests + - pytest + cache: + key: + files: + - requirements.txt + paths: + - .cache/pip diff --git a/docs/pipelines/lang/python/index.rst b/docs/pipelines/lang/python/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..1b3669c3d7dccf3d60ef06ee2fcd50ae25e5e2b9 --- /dev/null +++ b/docs/pipelines/lang/python/index.rst @@ -0,0 +1,33 @@ +.. _pipelines-lang-python: + +====== +Python +====== + +This is a "sub-pipeline" for building, testing and linting new changes that have been made to `python `__ files of a specific GitLab project. + +.. toctree:: + :maxdepth: 1 + :hidden: + + build.rst + linters.rst + +:doc:`build` + A document which covers all CI jobs responsible for building and testing python code. + +:doc:`linters` + A document which covers all CI jobs responsible for linting python code. + +Usage +===== + +In order to use this "sub-pipeline", you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/lang/python/linters.rst b/docs/pipelines/lang/python/linters.rst new file mode 100644 index 0000000000000000000000000000000000000000..222b8d72f9050a84e51b4134164fb01d39aa4916 --- /dev/null +++ b/docs/pipelines/lang/python/linters.rst @@ -0,0 +1,119 @@ +======= +Linters +======= + +This is a "sub-pipeline" for linting new changes that have been made to python files of a specific GitLab project. + +.. _pipelines-lang-python-linters-triggers: + +Triggers +======== + +All jobs from *python "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to python files. +Right now such files are detected by these patterns: + +- ``**/*.py``. + +Jobs overview +============= + +check:python:black +------------------ + +This job executes `black `__ *only on changed python files* (not diff, but files) which will diagnose and fix style violations. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:python:flake8 +------------------- + +This job executes `flake8 `__ *only on changed python files* (not diff, but files) which will diagnose and fix style violations, or bugs that can be deduced via static analysis. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.flake8`` name. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:python:isort +------------------ + +This job executes `isort `__ *only on changed python files* (not diff, but files) which will diagnose and fix issues related to python imports. + +Different configuration options can be found on `official page `__ + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + ``isort`` for each input file will try to find the ``.isort.cfg`` file located in the closest parent directory of the input file. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + + * - BPROTO_CI_LINTER_TRACE + - Optional + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. diff --git a/docs/pipelines/lang/shell.rst b/docs/pipelines/lang/shell.rst new file mode 100644 index 0000000000000000000000000000000000000000..9668307796f8429ebb9fbfc05971bf37b27a70f1 --- /dev/null +++ b/docs/pipelines/lang/shell.rst @@ -0,0 +1,113 @@ +.. _pipelines-lang-shell: + +===== +Shell +===== + +This is a "sub-pipeline" for linting new changes that have been made to shell scripts of a specific GitLab project. + +.. _pipelines-lang-shell-triggers: + +Triggers +======== + +All jobs from *shell "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to shell scripts. +Right now such files are detected by these patterns: + +- ``**/*.sh``. + +Jobs overview +============= + +check:shell:format +------------------ + +This job executes `shfmt `__ *only on changed Shell files* (not diff, but files) which will diagnose and fix style violations, or bugs that can be deduced via static analysis. + +*ARTIFACTS* + Job will generate a patch file with corrected lines. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.editorconfig`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Type + + * - BPROTO_CI_LINTER_TRACE + - Optional + - Public + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - Internal + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +check:shell:lint +---------------- + +This job executes `shellcheck `__ *only on changed Shell files* (not diff, but files) which will diagnose and fix style violations like wrong variable and function names. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.shellcheck`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Type + + * - BPROTO_CI_LINTER_TRACE + - Optional + - Public + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - Internal + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Usage +===== + +In order to use above CI jobs, you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/lang/yaml.rst b/docs/pipelines/lang/yaml.rst new file mode 100644 index 0000000000000000000000000000000000000000..5b4c4aa763d38310fa2cb2c50d165de6dc19afcc --- /dev/null +++ b/docs/pipelines/lang/yaml.rst @@ -0,0 +1,73 @@ +.. _pipelines-lang-yaml: + +==== +YAML +==== + +This is a "sub-pipeline" for linting new changes that have been made to YAML files of a specific GitLab project. + +.. _pipelines-lang-yaml-triggers: + +Triggers +======== + +All jobs from *YAML "sub-pipeline"* stages are triggered only on merge requests and only if changes have been made to YAML files. +Right now such files are detected by these patterns: + +- ``**/*.yml``. + +Jobs overview +============= + +check:yaml:format +----------------- + +This job executes `yamllint `__ *only on changed YAML files* (not diff, but files) which will diagnose and fix style violations. + +*ARTIFACTS* + Job will generate a simple log file with all found issues. + +*CONFIG* + Configuration file is located at the root of the repository and has ``.yamllint`` name. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + + * - Name + - Necessity + - Type + + * - BPROTO_CI_LINTER_TRACE + - Optional + - Public + + * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES + - Mandatory + - Internal + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-check-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +Usage +===== + +In order to use above CI jobs, you need to add the following code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/linters_pipeline.rst b/docs/pipelines/linters_pipeline.rst deleted file mode 100644 index 39a189e7a7c74cb8aa82d67b068112aab1250ae6..0000000000000000000000000000000000000000 --- a/docs/pipelines/linters_pipeline.rst +++ /dev/null @@ -1,673 +0,0 @@ -================ -Linters Pipeline -================ - -This is a pipeline for running different linters on all available type of files to check them for consistency of all our code conventions, and to detect possible issues as fast as it's possible. - -Flowchart -========= - -This is a very generalized flowchart that combines in itself all possible roots of pipeline generation and execution. - -.. - Flowchart has been generated with the help of https://app.diagrams.net/. - If you need to change flowchart: - 1. Go to the mentioned site. - 2. Import _static/cicd/linters_pipeline_flowchart.drawio. - 3. Change flowchart - 4. Save as .drawio and .png files. - 5. Update appropriate files in _static folder. - -.. image:: /_static/cicd/linters_pipeline_flowchart.png - :target: ../_images/cicd/linters_pipeline_flowchart.png - :alt: Linters Pipeline Flowchart - :class: only-light - -.. image:: /_static/cicd/linters_pipeline_flowchart.dark.png - :target: ../_images/cicd/linters_pipeline_flowchart.dark.png - :alt: Linters Pipeline Flowchart - :class: only-dark - -Stages overview -=============== - -Check stage ------------ - -This is a stage for running different linters on appropriate set of modified files. - -To avoid duplication of already known information please take a look at **check stage** overview from* :ref:`overview-check-stage`. - -However to avoid going back and force some of the most important information still will be put here. - -All jobs from this stage are triggered only on **merge requests** and if a certain part of the projects have been modified (this can be C++, Docker, Shell files and so on...). - -All jobs are not allowed to fail as all project code must follow established conventions and contain no know defects and issues. - -check:cmake:format -^^^^^^^^^^^^^^^^^^ - -This job executes `cmake-format `__ *only on changed CMake files* (not diff, but files) which will reformat code accordingly to the established project code style. - -Configuration file is located at the root of the repository and has ``.cmake-format.py`` name. - -*ALLOW FAILURE* - .. warning:: This job is allowed to fail as it's still in development. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``*/CMakeLists.txt``, - - ``**/*.cmake``. - -check:cmake:lint -^^^^^^^^^^^^^^^^ - -This job executes `cmake-lint `__ *only on changed CMake files* (not diff, but files) which will diagnose and fix style violations like wrong variable and function names. - -Configuration file is located at the root of the repository and has ``.cmake-format.py`` name. - -*ALLOW FAILURE* - .. warning:: This job is allowed to fail as it's still in development. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``*/CMakeLists.txt``, - - ``**/*.cmake``. - -check:code:license -^^^^^^^^^^^^^^^^^^ - -This job executes `licenseheaders `__ on project source code files, which will detect incorrect license headers. - -Configuration file is located at the root of the repository and has ``.license.tmpl`` name. - -*ALLOW FAILURE* - .. warning:: This job is allowed to fail as it's still in development. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and on all project files except ``3rd-party`` folder. - -check:code:spelling -^^^^^^^^^^^^^^^^^^^ - -This job executes `codespell `__ on project source code files, which will detect common misspellings. - -Configuration file is located at the root of the repository and has ``.codespellrc`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and on all project files except ``3rd-party`` folder. - -check:commits:spelling -^^^^^^^^^^^^^^^^^^^^^^ - -This job executes `codespell `__ on all found branch commits, which will detect common misspellings. - -Configuration file is located at the root of the repository and has ``.codespellrc`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - Origin - - * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - Mandatory - - GitLab - -*TRIGGERS* - Job will be triggered only on **merge requests**. - -.. _linters-pipeline-check-cpp-coverage: - -check:cpp:coverage -^^^^^^^^^^^^^^^^^^ - -This job calculates an overall coverage of C++ source code by tests. -In order to do that job builds project with compiler specific flags which allow to generate files that can be processed by `Gcov `__. - -At this moment job doesn't reuse artifacts from *build* or *test* stage. -Even so it means pipeline could take much less time, it still much easy to manage a job by itself. - -*ARTIFACTS* - Job will generate a set of files suitable for generating an HTML report. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_CMAKE_GLOBAL_ARGS - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``project/**/*.cpp``, - - ``project/**/*.hpp``, - - ``project/**/*.ipp``. - -check:cpp:format -^^^^^^^^^^^^^^^^ - -This job executes `clang-format `__ *only on changes from C++ files* which will reformat code accordingly to the established project code style. - -``clang-format`` for each input file will try to find the ``..clang-format`` file located in the closest parent directory of the input file. - -Different configuration options can be found on `official page `__ - -*ALLOW FAILURE* - .. warning:: This job is allowed to fail as it's still in development. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - Origin - - * - CI_COMMIT_SHA - - Mandatory - - GitLab - - * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - Mandatory - - GitLab - - * - BPROTO_CI_LINTER_TRACE - - Optional - - - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - - - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``project/**/*.cpp``, - - ``project/**/*.hpp``, - - ``project/**/*.ipp``. - -check:cpp:include_guards -^^^^^^^^^^^^^^^^^^^^^^^^ - -This job executes ``checkguard`` from `guardonce `__ package *only on changed C++ files* (not diff, but files) which will detect wrong written `include guards `__. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - Origin - - * - CI_PROJECT_DIR - - Mandatory - - GitLab - - * - BPROTO_CI_LINTER_TRACE - - Optional - - - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - - - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``project/**/*.cpp``, - - ``project/**/*.hpp``, - - ``project/**/*.ipp``. - -.. _linters-pipeline-check-cpp-cppcheck: - -check:cpp:cppcheck -^^^^^^^^^^^^^^^^^^ - -This job executes `cppcheck `__ *only on changed C++ files* (not diff, but files) which detect bugs and focuses on detecting undefined behavior and dangerous coding constructs. - -Configuration file is located at the root of the repository and has ``.cppcheck`` name. - -*ARTIFACTS* - Job will generate a set of files suitable for generating an HTML report. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_CMAKE_GLOBAL_ARGS - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``project/**/*.cpp``, - - ``project/**/*.hpp``, - - ``project/**/*.ipp``. - -.. _linters-pipeline-check-cpp-tidy: - -check:cpp:tidy -^^^^^^^^^^^^^^ - -This job executes `clang-tidy `__ *only on a diff* which will diagnose and fix typical programming errors, like style violations, interface misuse, or bugs that can be deduced via static analysis. - -Configuration file is located at the root of the repository and has ``.clang-tidy`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - Origin - - * - CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - Mandatory - - GitLab - - * - BPROTO_CI_LINTER_TRACE - - Optional - - - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - - - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``project/**/*.cpp``, - - ``project/**/*.hpp``, - - ``project/**/*.ipp``. - -check:cpp:tidy (nightly) -^^^^^^^^^^^^^^^^^^^^^^^^ - -This is almost the same job as :ref:`linters-pipeline-check-cpp-tidy` with one slight difference. -This is a nightly job which checks entire code base on a daily bases. - -The main reason for such separation is ability to detect a set of issues that is impossible to find while only checking files diff. -The downside of this solution is a long running job, which can take more than half an hour to run. - -check:docker:lint -^^^^^^^^^^^^^^^^^ - -This job executes `hadolint `__ *only on changed Docker files* (not diff, but files) which will diagnose bugs that can be deduced via static analysis. - -Configuration file is located at the root of the repository and has ``.hadolint.yaml`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/.Dockerfile``. - -check:python:black -^^^^^^^^^^^^^^^^^^ - -This job executes `black `__ *only on changed python files* (not diff, but files) which will diagnose and fix style violations. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.py``. - -check:python:flake8 -^^^^^^^^^^^^^^^^^^^ - -This job executes `flake8 `__ *only on changed python files* (not diff, but files) which will diagnose and fix style violations, or bugs that can be deduced via static analysis. - -Configuration file is located at the root of the repository and has ``.flake8`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.py``. - -check:python:isort -^^^^^^^^^^^^^^^^^^ - -This job executes `isort `__ *only on changed python files* (not diff, but files) which will diagnose and fix issues related to python imports. - -``isort`` for each input file will try to find the ``.isort.cfg`` file located in the closest parent directory of the input file. - -Different configuration options can be found on `official page `__ - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.py``. - -check:shell:format -^^^^^^^^^^^^^^^^^^ - -This job executes `shfmt `__ *only on changed Shell files* (not diff, but files) which will diagnose and fix style violations, or bugs that can be deduced via static analysis. - -Configuration file is located at the root of the repository and has ``.editorconfig`` name. - -*ARTIFACTS* - Job will generate a patch file with corrected lines. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.sh``. - -check:shell:lint -^^^^^^^^^^^^^^^^ - -This job executes `shellcheck `__ *only on changed Shell files* (not diff, but files) which will diagnose and fix style violations like wrong variable and function names. - -Configuration file is located at the root of the repository and has ``.shellcheck`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.sh``. - -check:yaml:format -^^^^^^^^^^^^^^^^^ - -This job executes `yamllint `__ *only on changed YAML files* (not diff, but files) which will diagnose and fix style violations. - -Configuration file is located at the root of the repository and has ``.yamllint`` name. - -*ARTIFACTS* - Job will generate a simple log file with all found issues. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - * - BPROTO_CI_LINTER_TRACE - - Optional - - * - BPROTO_CI_MERGE_REQUEST_CHANGED_FILES - - Mandatory - -*TRIGGERS* - Job will be triggered only on **merge requests** and only if there were changes in the below files: - - - ``**/*.yml``. - -Report stage ------------- - -*See also report stage overview from* :ref:`overview-report-stage`. - -report:cpp:coverage -^^^^^^^^^^^^^^^^^^^ - -*ARTIFACTS* - Job produces a statically generated HTML reports using `Gcovr `__. - -*DEPENDENCIES* - Jobs require artifacts from :ref:`linters-pipeline-check-cpp-coverage` job. - -*TRIGGERS* - Jobs are triggered only on **merge requests** and only if :ref:`linters-pipeline-check-cpp-coverage` job has failed. - -report:cpp:cppcheck -^^^^^^^^^^^^^^^^^^^ - -*ARTIFACTS* - Job produces a statically generated HTML reports using ``cppcheck-htmlreport`` from the `cppcheck `__ package. - -*DEPENDENCIES* - Jobs require artifacts from :ref:`linters-pipeline-check-cpp-cppcheck` job. - -*TRIGGERS* - Jobs are triggered only on **merge requests** and only if :ref:`linters-pipeline-check-cpp-cppcheck` job has failed. - -Distribution -============ - -+--------------------------+---------------------------------------------------------+ -| Job | System Configuration | -+ +----------+----------+----------+--------------+---------+ -| | Alpine 3 | CentOS 7 | CentOS 8 | Ubuntu 20.04 | Windows | -+==========================+==========+==========+==========+==============+=========+ -| | .. centered:: **Check stage** | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cmake:format | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cmake:lint | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:code:license | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:code:spelling | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:commits:spelling | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:coverage | no | no | no | **yes** | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:format | Uses its own docker image: ``clang-format``. | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:include_guards | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:cppcheck | no | no | no | **yes** | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:tidy | no | no | no | **yes** | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:cpp:tidy (nightly) | no | no | no | **yes** | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:docker:lint | Uses its own docker image: ``hadolint``. | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:python:black | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:python:flake8 | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:python:isort | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:shell:format | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:shell:lint | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| check:yaml:format | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| | .. centered:: **Report stage** | -+--------------------------+----------+----------+----------+--------------+---------+ -| report:cpp:coverage | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ -| report:cpp:cppcheck | **yes** | no | no | no | no | -+--------------------------+----------+----------+----------+--------------+---------+ diff --git a/docs/pipelines/misc.rst b/docs/pipelines/misc.rst new file mode 100644 index 0000000000000000000000000000000000000000..bb7bc12e00741ba48be0d1a6c5f3db97be3436dc --- /dev/null +++ b/docs/pipelines/misc.rst @@ -0,0 +1,98 @@ +============= +Miscellaneous +============= + +This is a list of almost all available misc jobs that are currently run in our CI/CD pipelines. + +Jobs overview +============= + +.. _misc-jobs-remove-old-pipelines: + +remove_old_pipelines +-------------------- + +This job executes a custom script that removes outdated pipelines from project pipeline list. + +*ARTIFACTS* + Job doesn't generate any artifacts. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + :width: 100% + + * - Name + - Necessity + - Origin + + * - BPROTO_CI_PRIVATE_API_TOKEN + - Mandatory + - + + * - BPROTO_CI_ENABLE_PIPELINES_CLEANUP + - Mandatory + - + + * - BPROTO_CI_GENERIC_PACKAGES_REGISTRY + - Mandatory + - + + * - *CI_API_V4_URL* + - Mandatory + - GitLab + + * - *CI_PROJECT_ID* + - Mandatory + - GitLab + +*IMAGES* + :ref:`alpine:3 `. + +*STAGE* + :ref:`overview-stages-misc-stage`. + +*TRIGGERS* + Job should be triggered only by a scheduler. + +Scheduled job +^^^^^^^^^^^^^ + +For example GitLab scheduled job can be configured like this: + +.. list-table:: + :header-rows: 1 + :width: 100% + :widths: 40 60 + + * - Name + - Value + + * - Description + - Clean all CI pipelines older than 1 week + + * - Interval Pattern + - Custom (``0 0 * * 0``) + + * - Cron timezone + - ``[UTC 0] UTC`` + + * - Select target branch or tag + - ``main`` + + * - Variables + - ``BPROTO_CI_ENABLE_PIPELINES_CLEANUP`` is ``true`` + +Usage +===== + +In order to use above CI jobs, you need to add the follow code to your `.gitlab-ci.yml`: + +.. code-block:: yaml + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/lite.yml' diff --git a/docs/pipelines/misc_pipeline.rst b/docs/pipelines/misc_pipeline.rst deleted file mode 100644 index 32649adf40caabed25482f87dd571e478410812e..0000000000000000000000000000000000000000 --- a/docs/pipelines/misc_pipeline.rst +++ /dev/null @@ -1,48 +0,0 @@ -============= -Misc Pipeline -============= - -This is a list of almost all available misc jobs that are currently run in our CI/CD pipelines. - -Stages overview -=============== - -Misc stage ----------- - -To avoid duplication of already known information please take a look at **misc stage** overview from :ref:`overview-misc-stage`. - -.. _misc-jobs-remove-old-pipelines: - -remove_old_pipelines -^^^^^^^^^^^^^^^^^^^^ - -This job executes a custom script that removes outdated pipelines from project pipeline list. - -*ENVIRONMENT* - -.. list-table:: - :header-rows: 1 - - * - Name - - Necessity - - Origin - - * - CI_API_V4_URL - - Mandatory - - GitLab - - * - CI_PROJECT_ID - - Mandatory - - GitLab - - * - BPROTO_CI_PRIVATE_API_TOKEN - - Mandatory - - - - * - BPROTO_CI_GENERIC_PACKAGES_REGISTRY - - Mandatory - - - -*TRIGGERS* - Jobs are triggered only by a scheduler. diff --git a/docs/pipelines/packages/conan.rst b/docs/pipelines/packages/conan.rst new file mode 100644 index 0000000000000000000000000000000000000000..81b726fc36964692b796621acd8bf31459fc7a04 --- /dev/null +++ b/docs/pipelines/packages/conan.rst @@ -0,0 +1,163 @@ +.. _pipelines-packages-conan: + +===== +Conan +===== + +This is a "sub-pipeline" for building, testing and deploying `conan packages `__. + +.. _pipelines-packages-conan-triggers: + +Triggers +======== + +All jobs from docker pipeline stages are triggered only if changes have been made to *conan recipe* related files. +Right now such files are detected by these patterns: + +- ``conandata.yml``, +- ``conanfile.py``. + +**See also:** `Conan packages in the GitLab Package Registry `__. + +Jobs overview +============= + +build:conan +----------- + +This stage is responsible for building conan packages from a clean state to make sure there are no unexpected results. + +*ALLOW FAILURE* + Job is not allowed to fail as any failure here means fundamental problems in a specific conan recipe. + +*ARTIFACTS* + Job doesn't generate any artifacts. + +*DEPENDENCIES* + Job doesn't require artifacts from other jobs and has no dependencies from other stages or jobs. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + :width: 100% + + * - Name + - Necessity + - Origin + + * - BPROTO_CI_DEPLOY_USER + - Mandatory + - + + * - BPROTO_CI_DEPLOY_TOKEN + - Mandatory + - + + * - BPROTO_CI_CONAN_PACKAGE_NAME + - Mandatory + - + + * - BPROTO_CI_CONAN_PACKAGES_REGISTRY + - Mandatory + - + + * - *CI_COMMIT_REF_NAME* + - Mandatory + - GitLab + + * - *CI_DEFAULT_BRANCH* + - Mandatory + - GitLab + + * - *CI_PROJECT_ROOT_NAMESPACE* + - Mandatory + - GitLab + +*IMAGES* + :ref:`conan:latest `. + +*STAGE* + :ref:`overview-stages-build-stage`. + +*TRIGGERS* + Job is triggered only on **merge requests** when any file from :ref:`trigger list ` is changed. + +deploy:conan +------------ + +.. _package registry: https://docs.gitlab.com/ee/user/packages/package_registry + +This stage is responsible for deploying conan packages to the `package registry`_ defined in ``gitlab`` `remote `__ (which should usually be equal to :ref:`BPROTO_CI_CONAN_PACKAGES_REGISTRY ` environment variable). + +*ALLOW FAILURE* + Job is not allowed to fail as these jobs are run only on ``CI_DEFAULT_BRANCH`` after accepting a successful merge request! + +*ARTIFACTS* + New conan package will be deployed to the project `package registry`_. + +*DEPENDENCIES* + Job doesn't generate any artifacts. + +*ENVIRONMENT* + +.. list-table:: + :header-rows: 1 + :width: 100% + + * - Name + - Necessity + - Origin + + * - BPROTO_CI_DEPLOY_USER + - Mandatory + - + + * - BPROTO_CI_DEPLOY_TOKEN + - Mandatory + - + + * - BPROTO_CI_CONAN_PACKAGE_NAME + - Mandatory + - + + * - BPROTO_CI_CONAN_PACKAGES_REGISTRY + - Mandatory + - + + * - *CI_COMMIT_REF_NAME* + - Mandatory + - GitLab + + * - *CI_DEFAULT_BRANCH* + - Mandatory + - GitLab + + * - *CI_PROJECT_ROOT_NAMESPACE* + - Mandatory + - GitLab + +*IMAGES* + :ref:`conan:latest `. + +*STAGE* + :ref:`overview-stages-deploy-stage`. + +*TRIGGERS* + Job is triggered only on ``CI_DEFAULT_BRANCH`` when any file from :ref:`trigger list ` is changed. + +Usage +===== + +To correctly use provided CI jobs you need to set :ref:`BPROTO_CI_CONAN_PACKAGES_REGISTRY ` either in :gitlab-docs:ci:`project CI/CD settings ` or as shown below. + +.. code-block:: yaml + + variables: + BPROTO_CI_CONAN_PACKAGE_NAME: "test_package" + + include: + - project: 'bproto/cicd' + ref: main + file: + - '/.gitlab/ci/external/conan.yml' diff --git a/docs/pipelines/packages/index.rst b/docs/pipelines/packages/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a8feca6e0029f0fcfa1214ab6521ee96218adba1 --- /dev/null +++ b/docs/pipelines/packages/index.rst @@ -0,0 +1,14 @@ +======== +Packages +======== + +These documents cover in-detail all available "sub-pipelines" for generating different type of packages. + +.. toctree:: + :maxdepth: 1 + :hidden: + + conan.rst + +:doc:`conan` + A GitLab CI/CD "sub-pipeline" which covers all CI jobs responsible for building and deploying conan packages. diff --git a/docs/useful_links.rst b/docs/useful_links.rst index 50d4d07929521059a367fe755ed49136877058f1..72f1554c5de0339604a2a9b4d95836f40b648d72 100644 --- a/docs/useful_links.rst +++ b/docs/useful_links.rst @@ -8,32 +8,32 @@ CI/CD **.gitlab-ci.yml** Official GitLab documentation for ``.gitlab-ci.yml`` keyword reference. - Available online at https://docs.gitlab.com/ee/ci/yaml/. + Available online at :gitlab-docs:ci:`yaml`. **GitLab CI/CD pipelines** Official GitLab documentation for managing GitLab CI/CD pipelines. - Available online at https://docs.gitlab.com/ee/ci/pipelines. + Available online at :gitlab-docs:ci:`pipelines`. **GitLab CI/CD jobs** Official GitLab documentation for managing GitLab CI/CD jobs. - Available online at https://docs.gitlab.com/ee/ci/jobs. + Available online at :gitlab-docs:ci:`jobs`. **GitLab CI/CD variables** Official GitLab documentation for managing GitLab CI/CD variables. - Available online at https://docs.gitlab.com/ee/ci/variables. + Available online at :gitlab-docs:ci:`variables`. **GitLab CI/CD pre-defined variables** Official GitLab documentation for managing GitLab CI/CD predefined variables. - Available online at https://docs.gitlab.com/ee/ci/variables/predefined_variables.html. + Available online at :gitlab-docs:ci:`variables/predefined_variables.html`. **GitLab CI/CD caching** Official GitLab documentation for using caching in GitLab CI/CD. - Available online at https://docs.gitlab.com/ee/ci/caching. + Available online at :gitlab-docs:ci:`caching`. Useful artifacts ================ @@ -41,22 +41,22 @@ Useful artifacts **GitLab CI/CD artifacts reports types** Official list of all available GitLab CI/CD artifacts reports types. - Available online at https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html. + Available online at :gitlab-docs:ci:`yaml/artifacts_reports.html`. **Code Quality** Official GitLab documentation for managing code quality reports. - Available online at https://docs.gitlab.com/ee/ci/testing/code_quality.html. + Available online at :gitlab-docs:ci:`testing/code_quality.html`. **Test coverage** Official GitLab documentation for managing test coverage reports. - Available online at https://docs.gitlab.com/ee/ci/testing/test_coverage_visualization.html. + Available online at :gitlab-docs:ci:`testing/test_coverage_visualization.html`. **Unit tests** Official GitLab documentation for managing unit tests reports. - Available online at https://docs.gitlab.com/ee/ci/testing/unit_test_reports.html. + Available online at :gitlab-docs:ci:`testing/unit_test_reports.html`. Pages ===== diff --git a/docs/pipelines/variables.rst b/docs/variables.rst similarity index 73% rename from docs/pipelines/variables.rst rename to docs/variables.rst index cf93fd48d14f85e3c59553c110cc04eb8c616deb..d95bb4101166d6d0139a72613274a78f065ce782 100644 --- a/docs/pipelines/variables.rst +++ b/docs/variables.rst @@ -8,7 +8,7 @@ Local CI/CD variables ===================== **CI_DEBUG_TRACE**:STRING - Predefined GitLab CI variable that enables `debug logging (tracing) `__ to help troubleshoot problems with pipeline configuration or job scripts. + Predefined GitLab CI variable that enables :gitlab-docs:ci:`debug logging (tracing) ` to help troubleshoot problems with pipeline configuration or job scripts. Valid values: ``false`` and ``true``. @@ -19,13 +19,10 @@ Local CI/CD variables The output contains the content of all variables and other secrets available to the job. The output is uploaded to the GitLab server and visible in job logs. -**BPROTO_CI_CONAN_PACKAGE_NAME**:STRING - A name of a conan package which will be used when deploying to the `artifacts `__ package registry. - -**BPROTO_CI_DOCS_NAME**:STRING - A name of an archive with generated HTML documentation which will be deployed to the `artifacts `__ package registry. +.. _variables-bproto-ci-conan-package-name: - Usually defaults to one of ``BPROTO_CI_*_DOCS_NAME`` variables. +**BPROTO_CI_CONAN_PACKAGE_NAME**:STRING + A name of a conan package which will be used when deploying to the :ref:`artifacts ` package registry. **BPROTO_CI_LINTER_TRACE**:STRING Enable debug logging to help troubleshoot problems with linter jobs. @@ -41,14 +38,28 @@ Local CI/CD variables .. note:: This variable is only available for **linter** jobs. +.. _variables-bproto-ci-sphinx-source-dir: + +**BPROTO_CI_SPHINX_SOURCE_DIR**:PATH + Path to `Sphinx `__ source files. + + Defaults to ``docs``. + +.. _variables-bproto-ci-sphinx-warnings-as-errors: + +**BPROTO_CI_SPHINX_WARNINGS_AS_ERRORS**:BOOL + Turn ``sphinx-build`` warnings into errors. + + Defaults to ``false``. + Global CI/CD variables ====================== -**BPROTO_CI_CICD_DOCS_NAME**:STRING - A package name for `CI/CD `__ project HTML docs. +**BPROTO_CI_CONAN_PACKAGES_REGISTRY**:URL + A link to an :ref:`artifacts `'s `conan packages `__ `registry `__. **BPROTO_CI_DEPLOY_TOKEN**:STRING - A `deploy token `__ for `artifacts`_'s `packages and registries `__. + A `deploy token `__ for :ref:`artifacts `'s `packages and registries `__. **Scopes:** ``read_registry``, ``write_registry``, ``read_package_registry``, ``write_package_registry``. @@ -56,10 +67,10 @@ Global CI/CD variables A user for ``BPROTO_CI_DEPLOY_TOKEN``. **BPROTO_CI_GENERIC_PACKAGES_REGISTRY**:URL - A link to an `artifacts`_'s `generic packages repository `__. + A link to an :ref:`artifacts `'s `generic packages repository `__. **BPROTO_CI_IMAGES_REGISTRY**:URL - A link to an `artifacts`_'s `container registry `__. + A link to an :ref:`artifacts `'s `container registry `__. **BPROTO_CI_PRIVATE_API_TOKEN**:STRING A `private token `__ of ``BPROTO_CI_USER`` with read-write access rights for API calls. @@ -69,15 +80,13 @@ Global CI/CD variables **BPROTO_CI_USER**:STRING An artificial user with maintainer rights. -.. _artifacts: https://gitlab.com/bproto/artifacts - GitLab group-level variables ---------------------------- +----------+-------------------------------------+----------------------------+-----------+---------+--------------+ | Type | Key | Value | Protected | Masked | Environments | +==========+=====================================+============================+===========+=========+==============+ -| Variable | BPROTO_CI_CICD_DOCS_NAME | cicd_docs | **yes** | no | All | +| Variable | BPROTO_CI_CONAN_PACKAGES_REGISTRY | \- | **yes** | no | All | +----------+-------------------------------------+----------------------------+-----------+---------+--------------+ | Variable | BPROTO_CI_DEPLOY_TOKEN | \- | no | **yes** | All | +----------+-------------------------------------+----------------------------+-----------+---------+--------------+ @@ -93,7 +102,7 @@ GitLab group-level variables +----------+-------------------------------------+----------------------------+-----------+---------+--------------+ .. note:: - To change above environment variables go to `group CI/CD settings `__ to *Variables* section. + To change above environment variables please follow instructions provided by the official :gitlab-docs:ci:`group CI/CD settings ` user guide. Misc CI/CD variables ==================== diff --git a/patches/clang-format/adjust_header_regroup_logic.patch b/patches/clang-format/adjust_header_regroup_logic.patch index 3af2805bee35863b68c88c19f1301893a6ab6e53..94cd59c499b002b7e7ad025e1dd2beb129aeaa63 100644 --- a/patches/clang-format/adjust_header_regroup_logic.patch +++ b/patches/clang-format/adjust_header_regroup_logic.patch @@ -1,8 +1,8 @@ diff --git a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp -index fbceb26c3..7893153fe 100644 +index 15a2024c4..96b772d4b 100644 --- a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp +++ b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp -@@ -242,6 +242,12 @@ bool IncludeCategoryManager::isMainHeader(StringRef IncludeName) const { +@@ -243,6 +243,12 @@ bool IncludeCategoryManager::isMainHeader(StringRef IncludeName) const { // headers may not. StringRef HeaderStem = llvm::sys::path::stem(IncludeName); StringRef FileStem = llvm::sys::path::stem(FileName); // foo.cu for foo.cu.cc diff --git a/patches/clang-format/support_block_indent_for_braced_init.patch b/patches/clang-format/support_block_indent_for_braced_init.patch index be29d0022b6c1d0d139860a4a028d30541521de7..620250118251fcc114ba78d9dc82b63441610c90 100644 --- a/patches/clang-format/support_block_indent_for_braced_init.patch +++ b/patches/clang-format/support_block_indent_for_braced_init.patch @@ -1,13 +1,13 @@ -diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h -index f116a89ac..dcd4856b7 100644 ---- a/clang/lib/Format/FormatToken.h -+++ b/clang/lib/Format/FormatToken.h -@@ -698,7 +698,7 @@ public: - /// list that should be indented with a block indent. - bool opensBlockOrBlockTypeList(const FormatStyle &Style) const { - // C# Does not indent object initialisers as continuations. -- if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp()) -+ if (is(tok::l_brace) && getBlockKind() == BK_BracedInit) - return true; - if (is(TT_TemplateString) && opensScope()) - return true; +diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp +index d994ed048..c5871e360 100644 +--- a/clang/lib/Format/FormatToken.cpp ++++ b/clang/lib/Format/FormatToken.cpp +@@ -92,7 +92,7 @@ bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const { + + bool FormatToken::opensBlockOrBlockTypeList(const FormatStyle &Style) const { + // C# Does not indent object initialisers as continuations. +- if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp()) ++ if (is(tok::l_brace) && getBlockKind() == BK_BracedInit) + return true; + if (is(TT_TemplateString) && opensScope()) + return true; diff --git a/patches/clang-format/support_block_indent_for_template_closer.patch b/patches/clang-format/support_block_indent_for_template_closer.patch index 94c2df4f4fced47a22525fedeff15cd49abe67be..020443f80a9d4523e07b6956535042693b72d271 100644 --- a/patches/clang-format/support_block_indent_for_template_closer.patch +++ b/patches/clang-format/support_block_indent_for_template_closer.patch @@ -1,53 +1,53 @@ diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp -index 45a4d2355..a767a2987 100644 +index 0ca297a5f..2dba2e8dd 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp -@@ -341,7 +341,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { - if (State.Stack.back().BreakBeforeClosingBrace && - Current.closesBlockOrBlockTypeList(Style)) +@@ -367,7 +367,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) { + Current.isBlockIndentedInitRBrace(Style)))) { return true; -- if (State.Stack.back().BreakBeforeClosingParen && Current.is(tok::r_paren)) -+ if (State.Stack.back().BreakBeforeClosingParen && Current.isOneOf(tok::r_paren, TT_TemplateCloser)) + } +- if (CurrentState.BreakBeforeClosingParen && Current.is(tok::r_paren)) ++ if (CurrentState.BreakBeforeClosingParen && Current.isOneOf(tok::r_paren, TT_TemplateCloser)) return true; - if (Previous.is(tok::semi) && State.LineContainsContinuedForLoopSection) - return true; -@@ -952,7 +952,7 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State, - opensProtoMessageField(*PreviousNonComment, Style))) - State.Stack.back().BreakBeforeClosingBrace = true; - -- if (PreviousNonComment && PreviousNonComment->is(tok::l_paren)) -+ if (PreviousNonComment && PreviousNonComment->isOneOf(tok::l_paren, TT_TemplateOpener)) - State.Stack.back().BreakBeforeClosingParen = + if (Style.Language == FormatStyle::LK_ObjC && + Style.ObjCBreakBeforeNestedBlockParam && +@@ -1050,7 +1050,7 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State, + CurrentState.BreakBeforeClosingBrace = true; + } + +- if (PreviousNonComment && PreviousNonComment->is(tok::l_paren)) { ++ if (PreviousNonComment && PreviousNonComment->isOneOf(tok::l_paren, TT_TemplateOpener)) { + CurrentState.BreakBeforeClosingParen = Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent; - -@@ -1051,7 +1051,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { - Current.Next->isOneOf(tok::semi, tok::kw_const, tok::l_brace))) + } +@@ -1170,7 +1170,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { return State.Stack[State.Stack.size() - 2].LastSpace; + } if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent && -- Current.is(tok::r_paren) && State.Stack.size() > 1) -+ Current.isOneOf(tok::r_paren, TT_TemplateCloser) && State.Stack.size() > 1) - return State.Stack[State.Stack.size() - 2].LastSpace; - if (NextNonComment->is(TT_TemplateString) && NextNonComment->closesScope()) - return State.Stack[State.Stack.size() - 2].LastSpace; +- (Current.is(tok::r_paren) || ++ (Current.isOneOf(tok::r_paren, TT_TemplateCloser) || + (Current.is(tok::r_brace) && + Current.MatchingParen->is(BK_BracedInit))) && + State.Stack.size() > 1) { diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp -index 9d130dbb0..e118ca4cf 100644 +index 73840332e..a7317fba2 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp -@@ -4322,8 +4322,6 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, +@@ -5482,8 +5482,6 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, if (Right.is(TT_ImplicitStringLiteral)) return false; - + - if (Right.is(TT_TemplateCloser)) - return false; if (Right.is(tok::r_square) && Right.MatchingParen && - Right.MatchingParen->is(TT_LambdaLSquare)) + Right.MatchingParen->is(TT_LambdaLSquare)) { return false; -@@ -4334,7 +4332,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, - return Right.MatchingParen && Right.MatchingParen->is(BK_Block); - - // We only break before r_paren if we're in a block indented context. -- if (Right.is(tok::r_paren)) { +@@ -5491,7 +5489,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line, + + // We only break before r_brace if there was a corresponding break before + // the l_brace, which is tracked by BreakBeforeClosingBrace. +- if (Right.is(tok::r_brace)) { + if (Right.isOneOf(tok::r_paren, TT_TemplateCloser)) { - if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent) { - return Right.MatchingParen && - !(Right.MatchingParen->Previous && + return Right.MatchingParen && (Right.MatchingParen->is(BK_Block) || + (Right.isBlockIndentedInitRBrace(Style))); + } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..8d399000d0a82a029a914b56ec274259c108ddbe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[project] +name = "3ct-packages" +version = "0.0.0" +requires-python = ">=3.8" + +dependencies = [ + # TODO(b110011): 'pyyaml' is required by 'conan' and 'yamllint', but there + # is an issue occurred while resolving dependencies: + # - https://github.com/yaml/pyyaml/issues/601 + # For time being let's put this here. + "pyyaml~=5.3.1", +] + +[tool.setuptools] +packages = [] + +[project.optional-dependencies] +conan = [ + "conan==1.59.0", + # "conan==2.0.2", # TODO(b110011): Replace when migration to Conan 2 is finished. +] +docs = [ + # base + "furo==2022.9.29", + "sphinx==4.5.0", + # c++ extensions + "breathe==4.32.0", + "exhale==0.2.4", + # extensions + "sphinx-copybutton", + "sphinx-design", + "sphinx-prompt==1.5.0", # NOTE(b110011): >=1.6.0 requires Sphinx<7.0.0,>=6.0.0 + # resolving dependencies + "docutils==0.17.1", + "pygments==2.11.0", +] +linters = [ + "black", # python + "codespell", # common + "cmakelang", # cmake + # "cppcheck_codequality", # c++ + # "cppcheck_junit", # c++ + "flake8", # python + "flake8-bandit", # python + "flake8-builtins", # python + "flake8-bugbear", # python + "flake8-comprehensions", # python + "gcovr", # c++ + "guardonce", # c++ + "isort", # python + "licenseheaders", # common + "pep8-naming", # python + "rstcheck", # rst + "yamllint", # yaml +] +misc = [ + "dateparser", + "requests", +] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 75363011a2daa1048be09db17c049123a5b07619..0000000000000000000000000000000000000000 --- a/requirements.txt +++ /dev/null @@ -1,51 +0,0 @@ -# Build - -conan==1.59.0 -# TODO(szinchen): Replace when migration to Conan 2 is finished. -#conan==2.0.2 - -# Documentation - -breathe==4.32.0 -exhale==0.2.4 - -furo==2022.9.29 -sphinx==4.5.0 - -sphinx-design -sphinx-jsonschema -sphinx-prompt==1.5.0 # >=1.6.0 requires Sphinx<7.0.0,>=6.0.0 - -# See https://github.com/sphinx-contrib/openapi/issues/122 -mistune==0.8.4 -sphinxcontrib-openapi==0.7.0 - -# Linters - -black # python -codespell # common -cmakelang # cmake -# cppcheck_codequality # c++ -# cppcheck_junit # c++ -flake8 # python -flake8-bandit # python -flake8-builtins # python -flake8-bugbear # python -flake8-comprehensions # python -gcovr # c++ -guardonce # c++ -isort # python -licenseheaders # common -pep8-naming # python -rstcheck # rst -yamllint # yaml - -# Misc - -dateparser -requests - -# Resolving dependencies - -docutils==0.17.1 -pygments==2.11.0 diff --git a/scripts/cicd/lint/check_commit_messages.sh b/scripts/cicd/lint/check_commit_messages.sh index 5188da59562dd81d54886834d50afc634923ab68..f77257516131be36304f6aedc6f38598b9c7307a 100644 --- a/scripts/cicd/lint/check_commit_messages.sh +++ b/scripts/cicd/lint/check_commit_messages.sh @@ -37,6 +37,52 @@ elif [[ -n "${BPROTO_CODESPELL_CFG}" ]]; then declare -r SETTINGS_FILE="${BPROTO_CODESPELL_CFG}" fi +# FMI: +# - https://www.jessesquires.com/blog/2017/08/08/customizing-git-log/ +# - https://stackoverflow.com/questions/33806068 +# - https://stackoverflow.com/questions/76120184 +declare -r GIT_PRETTY_LOG_FORMAT=' +commit %C(bold magenta)%H%Creset +Author: %C(bold blue)%an%Creset %C(blue)<%ae>%Creset +Date: %C(bold cyan)%cd%Creset + +%w(120,4,4)%-B%Creset +' + +####################################### +# Extract a trailer part of a commit message with a specified key. +# Arguments: +# $1 - a trailer key +# $2 - a commit hash +# Outputs: +# A content of a specific trailer with new lines and indentation removed. +####################################### +get_trailer() +{ + local -r key="${1}" + local -r commit="${2}" + + # NOTE(b110011): Remove new lines and make one long string. + # If indentation is wrong, we will still get an error when grepping. + git log --format="%(trailers:key=${key})" -n1 "${commit}" | sed "s/ \n/,/g" +} + +####################################### +# Sort line values in ascending order. +# Arguments: +# $1 - input line +# $2 - values delimiter (default: ', ') +# Outputs: +# A line with sorted values. +####################################### +sort_inplace() +{ + local -r line="${1}" + local -r delim="${2:-, }" + + echo "${line}" | sed "s|${delim}|\n|g" | sort | sed ':a;N;$!ba;s|\n|'"${delim}"'|g' +} + ####################################### # Extract all commits from HEAD of current branch to its parent and check them # for typos. @@ -53,6 +99,7 @@ fi check() { local -r target_branch="${1}" + local -i failed=0 git fetch origin "${target_branch}" @@ -63,6 +110,18 @@ check() for commit in $(git rev-list "origin/${target_branch}..HEAD"); do git log -n 1 "${commit}" > "${COMMIT_MSG_FILE}" + local -i is_commit_message_logged=0 + log_commit_message() + { + failed=1 # If we need to call this method, we have hit a jackpot. + + if [[ ${is_commit_message_logged} -eq 0 ]]; then + is_commit_message_logged=1 + + git log --color=always --pretty=format:"${GIT_PRETTY_LOG_FORMAT}" -n1 "${commit}" + fi + } + codespell \ --config "${SETTINGS_FILE}" \ --enable-colors \ @@ -70,11 +129,72 @@ check() > "${CODESPELL_LOG_FILE}" || true if [[ -s "${CODESPELL_LOG_FILE}" ]]; then - cat "${COMMIT_MSG_FILE}" - echo "" + log_commit_message cat "${CODESPELL_LOG_FILE}" echo "" fi + + # FMI: https://unix.stackexchange.com/questions/42898 + # NOTE(b110011): Doing as shellcheck suggests breaking the script. + # shellcheck disable=SC2155 + local long_lines="$(git log --pretty=format:"%B" -n1 "${commit}" | grep '^.\{80\}' --color=always)" + if [[ -n "${long_lines}" ]]; then + log_commit_message + echo -e "\033[0;31mBelow lines contain more than 80 symbols:\033[0m" + echo " - ${long_lines//$'\n'/$'\n' - }" # Prepend ' - ' to each line. + fi + + check_trailer() + { + local -r key="${1}" + local -r regex="${2}" + local -r message="${3}" + local -r allowed_format="${4}" + + local -r line="$(get_trailer "${key}" "${commit}")" + if [[ -n "${line}" ]]; then + local -r matched_line="$(grep -Eo "${regex}" <<< "${line}")" + if [ "${matched_line}" != "${line}" ]; then + log_commit_message + echo -e "\033[0;31mInvalid format of ${message}! \033[0m" + echo -e " \033[0;33mAllowed format: ${allowed_format}\033[0m" + + return # Stop here because we might not be able to correctly perform next check. + fi + + local -r trailer_values=${line#"${key}: "} # Remove key from line. + local -r sorted_trailer_values="$(sort_inplace "${trailer_values}")" + if [[ "${sorted_trailer_values}" != "${trailer_values}" ]]; then + log_commit_message + echo -e "\033[0;31m${message^} are not properly sorted! \033[0m" + echo -e " \033[0;33mCorrent order: ${sorted_trailer_values}\033[0m" + fi + fi + } + + check_trailer \ + 'GitHub' \ + 'GitHub: (#[0-9]+(, ){0,}){0,}' \ + 'GitHub issue references' \ + 'GitHub: #1, #2' + + check_trailer \ + 'GitLab' \ + 'GitLab: (#[0-9]+(, ){0,}){0,}' \ + 'GitLab issue references' \ + 'GitLab: #1, #2' + + check_trailer \ + 'Jira' \ + 'Jira: ([a-zA-Z]+-[0-9]+(, ){0,}){0,}' \ + 'JIRA ticket IDs' \ + 'Jira: MO-1, MO-10' + + check_trailer \ + 'Tags' \ + 'Tags: ([a-zA-Z]+(, ){0,}){0,}' \ + 'commit message tags' \ + 'Tags: TagA, TagB' done # Restore output to terminal. @@ -91,7 +211,7 @@ check() # Fail if there are some changes # shellcheck disable=SC2046 - exit $(grep -c "commit.txt" "${LOG_FILE}") + exit ${failed} } ####################################### diff --git a/scripts/cicd/lint/check_cpp_conventions.sh b/scripts/cicd/lint/check_cpp_conventions.sh index 6c7d4bdc5258fac920da3c554fa6b728bbf1d178..3f4b036a8789b05be3ad214ae4afd7f58ac312b8 100644 --- a/scripts/cicd/lint/check_cpp_conventions.sh +++ b/scripts/cicd/lint/check_cpp_conventions.sh @@ -51,7 +51,7 @@ if [[ -s "${DEFAULT_CLANGTIDY_CFG}" ]]; then elif [[ -n "${BPROTO_CLANGTIDY_CFG}" ]]; then declare -r CLANGTIDY_CFG="${BPROTO_CLANGTIDY_CFG}" - # TODO(szinchen): This is a workaround for currunt clang-tidy scripts. + # TODO(b110011): This is a workaround for currunt clang-tidy scripts. # Even so 'run-clang-tidy' supports '-config-file', 'clang-tidy-diff' # still doesn't know about this. Therefore it's better to use same # approach for both tools. @@ -64,7 +64,7 @@ else exit 1 fi -# NOTE(szinchen): gmock in gtest/1.11.0 generated some errors in "merge request" +# NOTE(b110011): gmock in gtest/1.11.0 generated some errors in "merge request" # mode, so we want to disable them and clean our pipeline. declare -r DISABLED_GMOCK_CHECKS=( "misc-non-private-member-variables-in-classes" @@ -184,7 +184,12 @@ check_all() # Perform check - run-clang-tidy-14 -j "${NPROC}" -p "${CMAKE_BUILD_DIR}" -quiet | tee "${LOG_FILE}" || true + run-clang-tidy \ + -j "${NPROC}" \ + -p "${CMAKE_BUILD_DIR}" \ + -quiet \ + -use-color \ + | tee "${LOG_FILE}" || true # Deploy diff --git a/scripts/cicd/lint/check_include_guards.sh b/scripts/cicd/lint/check_cpp_include_guards.sh similarity index 95% rename from scripts/cicd/lint/check_include_guards.sh rename to scripts/cicd/lint/check_cpp_include_guards.sh index 454b71df49f2e87019367e85feb281c323be63cd..3a9551609dada396921327104056c78f68fe70ae 100644 --- a/scripts/cicd/lint/check_include_guards.sh +++ b/scripts/cicd/lint/check_cpp_include_guards.sh @@ -16,7 +16,7 @@ # # DESCRIPTION: # -# Check C++ header file for correct include guard with help of a guardonce. +# Extract C++ header files and check for correct format of include guards. # # ATTENTION: # @@ -76,7 +76,7 @@ check_api_file() local -r dir_type="$(echo "${file}" | grep -Eo "${SPECIAL_DIRS}")" # https://wiki.bash-hackers.org/syntax/pe - local -r project="${file%%/${dir_type}*}" # Get path before dir_type + local -r project="${file%%/"${dir_type}"*}" # Get path before dir_type # In CI mode we get sorted list of modified files, so we can optimise this a # little bit. @@ -85,7 +85,7 @@ check_api_file() cd "${destination}" fi - local -r project_file="${file##*${dir_type}/}" # Get path after dir_type + local -r project_file="${file##*"${dir_type}"/}" # Get path after dir_type check_file "${log_file}" "${project_file}" } @@ -107,7 +107,7 @@ print_files() } ####################################### -# Main entry point for this script. +# Extract C++ files and check their include guards. # Globals: # API_DIRS # LOG_FILENAME diff --git a/scripts/cicd/lint/check_cpp_iostream.sh b/scripts/cicd/lint/check_cpp_iostream.sh new file mode 100644 index 0000000000000000000000000000000000000000..e1dd9d2ad8a910453aa1251b6da8a6c81e58ab43 --- /dev/null +++ b/scripts/cicd/lint/check_cpp_iostream.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DESCRIPTION: +# +# Extract C++ files and check for inclusion of iostream file. +# +# ATTENTION: +# +# This script is mainly supposed to be used only by CI/CD linter jobs. + +set -e + +####################################### +# Find all 'iosteam' includes in a file. +# Arguments: +# $1 - a file path +# Outputs: +# Writes all found typos to stdout and to a log file. +# Note: +# We don't handle ignored includes here. +####################################### +find_iostream() +{ + grep -Hn -e '#include ' -e '#include "iostream"' --color=always "${1}" +} + +####################################### +# Extract C++ files and check them for iostream file inclusion. +# Globals: +# BPROTO_CI_LINTER_TRACE +# Arguments: +# $1 - a list of files to be checked +# Outputs: +# Writes all found typos to stdout and to a log file. +####################################### +check() +{ + local -i found=0 + + # Extract C++ header files if any. + local -r changed_files="$(echo "${2}" | grep -E ".*\.[chi]pp$")" + + if [[ -n "${changed_files}" ]]; then + if [[ "${BPROTO_CI_LINTER_TRACE}" == "true" ]]; then + echo -e "\033[0;33mDetected changes in ${changed_files}\033[0m" + fi + + while IFS= read -r file; do + if find_iostream "${file}" | grep -v "// ignore"; then + found=1 + fi + done <<< "${changed_files}" + fi + + # Fail if there are some changes + exit ${found} +} + +####################################### +# Main entry point for this script. +# Globals: +# BPROTO_CI_MERGE_REQUEST_CHANGED_FILES +# Arguments: +# $1 - a list of files to be checked +####################################### +main() +{ + if [[ -n "${1}" ]]; then + check "${1}" + elif [[ -v BPROTO_CI_MERGE_REQUEST_CHANGED_FILES ]]; then + # Variable can exist but is empty. + if [[ -n "${BPROTO_CI_MERGE_REQUEST_CHANGED_FILES}" ]]; then + check "${BPROTO_CI_MERGE_REQUEST_CHANGED_FILES}" + else + echo -e "\033[0;33mNo useful changes have been detected in files!\033[0m" + fi + else + echo -e "\033[0;31mFailed to detect modified files!\033[0m" >&2 + exit 1 + fi +} + +main "$@" diff --git a/scripts/cicd/lint/check_python_imports.sh b/scripts/cicd/lint/check_python_imports.sh index 809ad78163c3c27b5e2243c52aba1cba17563e9b..784907982469ed6ec49a3519df66085911e27d80 100644 --- a/scripts/cicd/lint/check_python_imports.sh +++ b/scripts/cicd/lint/check_python_imports.sh @@ -58,7 +58,7 @@ check() bat --color=always -pp "${PATCH_FILE}" # Make generated patch more GITish - # NOTE(szinchen): To apply generated patch run: + # NOTE(b110011): To apply generated patch run: # git apply -p0 isort.patch sed -i -e 's/:before//g' -e 's/:after//g' "${PATCH_FILE}" diff --git a/scripts/cicd/lint/filter_files.sh b/scripts/cicd/lint/filter_files.sh index e54777b714d6bcea37f7da89361b2eb5c2fe1ec4..caa9d6804dbe506635a7baba0fa41ab8bd7bba4a 100644 --- a/scripts/cicd/lint/filter_files.sh +++ b/scripts/cicd/lint/filter_files.sh @@ -43,7 +43,7 @@ filter() modified_files="$(git diff-tree --name-only -r "HEAD..origin/${target_branch}" \ | grep -v '3rd-party' | sort)" - # Exclude all deleted filed. + # Exclude all deleted files. while IFS="" read -r file || [[ -n "${file}" ]]; do if [ -r "${file}" ]; then echo "${file}" diff --git a/scripts/conan/fetch.sh b/scripts/conan/fetch.sh index 1f48ef454bf31117f244681b192547ee01a2f9ae..b7919711a0498fac3c9ad454a7cc69db5f2cc022 100644 --- a/scripts/conan/fetch.sh +++ b/scripts/conan/fetch.sh @@ -42,7 +42,14 @@ fetch() echo -e "\033[0;33mFetching Conan dependencies for CC=${cc}, CXX=${cxx}, CMAKE_BUILD_TYPE=${type} configuration...\033[0m" - # NOTE(szinchen): We need both '-if' and '-of' due to + # FMI: https://stackoverflow.com/questions/75103211 + # shellcheck disable=SC2046 + if [ $(command -v "${cxx}" &> /dev/null ; echo $?) -ne 0 ]; then + echo -e "\033[0;31mCould not find compiler set in environment variable CXX (${cxx})! Skipping current setup! \033[0m" + return + fi + + # NOTE(b110011): We need both '-if' and '-of' due to # https://github.com/conan-io/conan/issues/11483. conan install . -if build -of build --build=missing \ -pr:b "${cc}" \ @@ -57,7 +64,7 @@ fetch() # https://stackoverflow.com/questions/50247671 conan remove "*" -s -f - # TODO(szinchen): Replace when migration to Conan 2 is finished. + # TODO(b110011): Replace when migration to Conan 2 is finished. # https://docs.conan.io/2/reference/commands/cache.html#conan-cache-clean # https://github.com/conan-io/conan/issues/11562 # conan cache clean "*" -b -d -s @@ -70,7 +77,7 @@ fetch() ####################################### main() { - # TODO(szinchen): Uncomment when migration to Conan 2 is finished. + # TODO(b110011): Uncomment when migration to Conan 2 is finished. # conan profile detect fetch 'clang' 'clang++' 'Debug' diff --git a/scripts/conan/setup.sh b/scripts/conan/setup.sh index 6e46d065b596b81919bb6c9c5936823954b27edd..545418e827fa1a03746684b76847d7d83e2f6020 100644 --- a/scripts/conan/setup.sh +++ b/scripts/conan/setup.sh @@ -73,8 +73,13 @@ main() { local -r prefix="${1}" + echo -e "\033[0;33mInit a default conan setup\033[0m" init_defaults + + echo -e "\033[0;33mSetup custom conan profiles\033[0m" setup_profiles "${prefix}" + + echo -e "\033[0;33mSetup custom conan remotes\033[0m" setup_remotes } diff --git a/scripts/fork.sh b/scripts/fork.sh new file mode 100644 index 0000000000000000000000000000000000000000..dc16844ec524c33effc47b2932625f180b7a39cf --- /dev/null +++ b/scripts/fork.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# +# CLI +# + +usage() +{ + echo "Usage: $0 [-t ] -p -r + + -t target branch + -p prefix for macro and include guards + -r a repo short path (CI_PROJECT_PATH) +" 1>&2 + exit 1 +} + +while getopts ":t:p:r:" o; do + case "${o}" in + t) + declare -r TARGET_BRANCH="${OPTARG}" + ;; + p) + declare -r PREFIX="${OPTARG}" + ;; + r) + declare -r REPO_PATH="${OPTARG}" + ;; + *) + usage + ;; + esac +done + +shift $((OPTIND - 1)) + +if [ -z "${PREFIX}" ] || [ -z "${REPO_PATH}" ]; then + usage +fi + +if [ -z "${TARGET_BRANCH}" ]; then + declare TARGET_BRANCH="develop" +fi + +# +# Utils +# + +commit() +{ + echo -e "\033[0;32m${1} \033[0m" + git commit -q --message "${1}" --trailer "Tags: ${2}" +} + +commit_if_changed() +{ + local -r file="${1}" + local -r commit_msg="${2}" + local -r commit_tags="${3}" + + if git diff --name-only | grep -q -e "${file}"; then + git add "${file}" + commit "${commit_msg}" "${commit_tags}" + else + echo -e "\033[0;33mHave not detected changes in '${file}' file, skipping.\033[0m" + fi +} + +# +# Checks +# + +# Check 1 + +if [[ $(git rev-parse --show-toplevel 2> /dev/null) != "$PWD" ]]; then + echo -e "\033[0;31m'$0' can be used only in the root of git repository! \033[0m" + exit 1 +fi + +# Check 2 + +if git show-ref -q --heads "${TARGET_BRANCH}"; then + echo -e "\033[0;31m'${TARGET_BRANCH}' branch exists locally! \033[0m" + exit 1 +fi + +if git ls-remote --exit-code --heads origin "${TARGET_BRANCH}"; then + echo -e "\033[0;31m'${TARGET_BRANCH}' branch exists remotely! \033[0m" + exit 1 +fi + +# +# Mandatory actions +# + +# Action 1 + +git checkout -b "${TARGET_BRANCH}" + +# Action 2 + +declare -r EXTERNAL_CI=".gitlab/ci/external" + +find "${EXTERNAL_CI}" -type f -exec sed -i -e "s|project: 'bproto/cicd'|project: '${REPO_PATH}'|g" {} \; +git add "${EXTERNAL_CI}" +commit "Replace project path from 'bproto/cicd' to '${REPO_PATH}' in CI external files" "CI, Cpp" + +# Action 3 + +sed -i "s/BPROTO/${PREFIX}/g" .clang-tidy +git add .clang-tidy +commit "Replace BPROTO with ${PREFIX} in clang-tidy config file" "Cpp, ClangTidy, Linters" + +# Action 4 + +declare -r IG_LINTER="scripts/cicd/lint/check_cpp_include_guards.sh" + +sed -i "s/ __BPROTO_ / __${PREFIX}_ /g" "${IG_LINTER}" +git add "${IG_LINTER}" +commit "Change include guards prefix from BPROTO to ${PREFIX}" "Cpp, Linters, Shell" + +# Action 5 + +commit_if_changed 'pyproject.toml' 'Add project specific python packages' 'Python' + +# Action 6 + +commit_if_changed 'conan/profiles' 'Update conan profiles' 'Cpp, Conan' + +# Action 7 + +commit_if_changed 'conan/conanfile.txt' 'Add project specific conan packages' 'Cpp, Conan' + +# +# Optional actions +# + +# Action 1 + +commit_if_changed '.clang-format' 'Update clang-format config' 'Cpp, ClangFormat, Linters' + +# Action 2 + +commit_if_changed '.clang-tidy' 'Update clang-tidy config' 'Cpp, ClangTidy, Linters' diff --git a/scripts/os/setup_alpine3.sh b/scripts/os/setup_alpine3.sh index db20d7b891b8725e0259624094beb707d49a3e36..e27ad0f0e99d2b455cb691eae22c7a8db43d1c36 100644 --- a/scripts/os/setup_alpine3.sh +++ b/scripts/os/setup_alpine3.sh @@ -51,4 +51,4 @@ rm -rf /var/cache/apk/* # Python # #------------------------------------------------------------------------------# -pip3 install --no-cache-dir -r 'requirements.txt' +pip3 install --no-cache-dir . '.[docs]' '.[linters]' '.[misc]' diff --git a/scripts/os/setup_centos7.sh b/scripts/os/setup_centos7.sh index a2f6e67a4f94191feb2efc9b802784a10c4b9161..55d74672c426d741e6e30493cd841232051c8c52 100644 --- a/scripts/os/setup_centos7.sh +++ b/scripts/os/setup_centos7.sh @@ -27,19 +27,14 @@ set -e #------------------------------------------------------------------------------# declare -a INSTALL_PKGS=( - "devtoolset-10-gcc" - "devtoolset-10-gcc-c++" - "devtoolset-10-gcc-gfortran" - "devtoolset-10-gdb" - "devtoolset-10-libasan-devel" - "devtoolset-10-liblsan-devel" - "devtoolset-10-libtsan-devel" - "devtoolset-10-libubsan-devel" - "devtoolset-10-make" - "doxygen" + # compilation + "devtoolset-11-gcc" + "devtoolset-11-gcc-c++" + "devtoolset-11-gcc-gfortran" + "devtoolset-11-gdb" + "devtoolset-11-make" + # essential "git" - "llvm-toolset-13.0-clang" - "llvm-toolset-13.0-llvm" "rh-python38-python" "rh-python38-python-devel" "rh-python38-python-pip" @@ -85,8 +80,8 @@ source /opt/rh/rh-python38/enable # Install packages -pip3 install --no-cache-dir -r 'requirements.txt' -pip3 install --no-cache-dir 'cmake==3.21.0' +pip3 install --no-cache-dir . '.[conan]' +pip3 install --no-cache-dir 'cmake==3.25.0' #------------------------------------------------------------------------------# # Setup environment # @@ -160,7 +155,6 @@ sed -i -e $'s/Linux:/Linux:\\\n CentOS:/g' "${HOME}/.conan/settings.yml" # Make SCL available globally in user environment. # https://stackoverflow.com/questions/55901985 cat << EOF >> "/etc/profile.d/scl_enable.sh" -source scl_source enable devtoolset-10 -source scl_source enable llvm-toolset-13.0 +source scl_source enable devtoolset-11 source scl_source enable rh-python38 EOF diff --git a/scripts/os/setup_centos8.sh b/scripts/os/setup_centos8.sh index 6f5e3345a4f086d1cec25a746e346cbbaded8eed..1ae85202463ff5da41cbbc472019e6f5d3a6ae6d 100644 --- a/scripts/os/setup_centos8.sh +++ b/scripts/os/setup_centos8.sh @@ -30,19 +30,15 @@ set -e # See https://stackoverflow.com/questions/61590926 declare -a INSTALL_PKGS=( - "clang" + # compilation "cmake" "gcc-toolset-10-gcc" "gcc-toolset-10-gcc-c++" "gcc-toolset-10-gcc-gfortran" "gcc-toolset-10-gdb" - "gcc-toolset-10-libasan-devel" - "gcc-toolset-10-liblsan-devel" - "gcc-toolset-10-libtsan-devel" - "gcc-toolset-10-libubsan-devel" - "git" - "llvm" "make" + # essential + "git" "python38-pip" "python38-devel" ) @@ -73,7 +69,7 @@ yum -y clean all --enablerepo='*' # Python # #------------------------------------------------------------------------------# -pip3 install --no-cache-dir -r 'requirements.txt' +pip3 install --no-cache-dir . '.[conan]' #------------------------------------------------------------------------------# # Setup environment # diff --git a/scripts/os/setup_conan.sh b/scripts/os/setup_conan.sh index c7945936d540294b4e25b109e9450b74a4056c3a..509be280818bc5c33815f6ac6a837d27dde695a0 100644 --- a/scripts/os/setup_conan.sh +++ b/scripts/os/setup_conan.sh @@ -27,12 +27,16 @@ set -e #------------------------------------------------------------------------------# declare -a INSTALL_PKGS=( - "doxygen" - "g++-10" - "gcc-10" - "git" + # compilation + "g++-11" + "gcc-11" "make" + # essential + "git" "python3-pip" + "python3-venv" + # cppdoc + "doxygen" ) # Install @@ -55,21 +59,29 @@ rm -rf /var/lib/apt/lists/* # GCC -update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 110 +update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ - --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-10 \ - --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-10 + --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 #------------------------------------------------------------------------------# # Python # #------------------------------------------------------------------------------# +if [[ -v DOCKER_PYTHON_VENV ]]; then + if [[ -n "${DOCKER_PYTHON_VENV}" ]]; then + python3 -m venv "${DOCKER_PYTHON_VENV}" + # shellcheck disable=SC1091 + source "${DOCKER_PYTHON_VENV}/bin/activate" + fi +fi + declare -a INSTALL_PY_PKGS=( - "cmake==3.21.0" + "cmake==3.25.0" "conan==1.59.0" # cppdoc "breathe==4.32.0" diff --git a/scripts/os/setup_ubuntu2004.sh b/scripts/os/setup_ubuntu2004.sh index b76fe1ec2f3ce9cd6eb0b4c8e8a791e762424bd3..26ebe56101add6100f3934b70671d291279e3c85 100644 --- a/scripts/os/setup_ubuntu2004.sh +++ b/scripts/os/setup_ubuntu2004.sh @@ -27,25 +27,15 @@ set -e #------------------------------------------------------------------------------# declare -a INSTALL_PKGS=( - "clang-12" - "clang-tidy-14" # external - "cmake" # external - "cppcheck" - "curl" - "doxygen" - "g++-10" - "gcc-10" - "git" - "graphviz" - "jq" - "libgtk-3-dev" - "libpython3.8-dev" - "lcov" - "llvm-12" + # compilation + "g++-11" + "gcc-11" "make" - "pkg-config" + # essential + "git" "python3-pip" - "valgrind" + # pybind11 + "libpython3-dev" ) # @@ -56,20 +46,15 @@ declare -a INSTALL_PKGS=( export DEBIAN_FRONTEND=noninteractive export TZ=Europe/Kiev +# We need to enable official Ubuntu Toolchain PPA to get new version of GCC. +# - https://stackoverflow.com/questions/67298443 +# - https://wiki.ubuntu.com/ToolChain#PPA_packages apt-get update -apt-get install --no-install-recommends -y ca-certificates gnupg wget - -# Install clang-*-14 -# FMI https://apt.llvm.org/. - -echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" > /etc/apt/sources.list.d/llvm.list -echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" >> /etc/apt/sources.list.d/llvm.list - -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +apt-get install --no-install-recommends -y software-properties-common +add-apt-repository ppa:ubuntu-toolchain-r/test # Install packages -apt-get update apt-get install --no-install-recommends -y "${INSTALL_PKGS[@]}" # @@ -86,27 +71,20 @@ rm -rf /var/lib/apt/lists/* # See https://stackoverflow.com/questions/7832892 # See https://stackoverflow.com/questions/67298443 -# Clang - -update-alternatives \ - --install /usr/bin/clang clang /usr/bin/clang-12 100 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 - # GCC -update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 110 +update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ - --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-10 \ - --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-10 + --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 #------------------------------------------------------------------------------# # Python # #------------------------------------------------------------------------------# -pip3 install --no-cache-dir -r 'requirements.txt' - -pip3 install --no-cache-dir 'cmake==3.21.0' +pip3 install --no-cache-dir . '.[conan]' +pip3 install --no-cache-dir 'cmake==3.25.0' diff --git a/scripts/os/setup_ubuntu2304.sh b/scripts/os/setup_ubuntu2304.sh new file mode 100644 index 0000000000000000000000000000000000000000..5dff5883d5b034e389c2a031c4a456c40642bb66 --- /dev/null +++ b/scripts/os/setup_ubuntu2304.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Copyright 2023 Zinchenko Serhii . +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DESCRIPTION: +# +# This script is mainly supposed to be used only by Ubuntu 20.04 docker image to +# provide a pre-build image with all pre-installed and pre-configured system +# packages and project dependencies. + +set -e + +#------------------------------------------------------------------------------# +# Packages # +#------------------------------------------------------------------------------# + +declare -a INSTALL_PKGS=( + # compilation + "clang-15" + "g++-11" + "gcc-11" + "llvm-15" + "make" + # essential + "git" + "python3-pip" + "python3-venv" + # linters / sanitizers + "clang-tidy-15" + "cppcheck" + "valgrind" + # utils + "curl" + "doxygen" + "jq" + # pybind11 + "libpython3-dev" +) + +# +# Install +# + +# set noninteractive installation +export DEBIAN_FRONTEND=noninteractive +export TZ=Europe/Kiev + +# Install packages + +apt-get update +apt-get install --no-install-recommends -y "${INSTALL_PKGS[@]}" + +# +# Clean +# + +apt-get clean +rm -rf /var/lib/apt/lists/* + +#------------------------------------------------------------------------------# +# Packages Aliases # +#------------------------------------------------------------------------------# + +# See https://stackoverflow.com/questions/7832892 +# See https://stackoverflow.com/questions/67298443 + +# Clang + +update-alternatives \ + --install /usr/bin/clang clang /usr/bin/clang-15 100 \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-15 + +# GCC + +update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +update-alternatives \ + --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 + +#------------------------------------------------------------------------------# +# Python # +#------------------------------------------------------------------------------# + +if [[ -v DOCKER_PYTHON_VENV ]]; then + if [[ -n "${DOCKER_PYTHON_VENV}" ]]; then + python3 -m venv "${DOCKER_PYTHON_VENV}" + # shellcheck disable=SC1091 + source "${DOCKER_PYTHON_VENV}/bin/activate" + fi +fi + +pip3 install --no-cache-dir . '.[conan]' '.[docs]' '.[linters]' '.[misc]' +pip3 install --no-cache-dir 'cmake==3.25.0'