Run benchmarks and tests on gitlab CI for windowsci_push_test

1 files changed, 290 insertions, 277 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87e6260f76..bf8cf01c65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,164 +3,164 @@
 stages:
   - build
 
-.Debian_Image:
-  tags:
-    - docker
-    - linux
-  image: debian:bullseye
-
-.Debian:
-  extends: .Debian_Image
-  cache:
-    paths:
-    - apt-cache/
-    - ccache/
-  stage: build
-  script:
-    - export CCACHE_BASEDIR="`pwd`"
-    - export CCACHE_DIR="`pwd`/ccache" && mkdir -pv "$CCACHE_DIR"
-    - ccache -z -M "${CCACHE_SIZE}"
-    - CI/before_script.linux.sh
-    - cd build
-    - cmake --build . -- -j $(nproc)
-    - cmake --install .
-    - if [[ "${BUILD_TESTS_ONLY}" ]]; then ./openmw_test_suite; fi
-    - if [[ "${BUILD_TESTS_ONLY}" ]]; then ./openmw_detournavigator_navmeshtilescache_benchmark; fi
-    - ccache -s
-  artifacts:
-    paths:
-      - build/install/
-
-Coverity:
-  extends: .Debian_Image
-  stage: build
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-  before_script:
-    - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-dynamic coverity
-    - curl -o /tmp/cov-analysis-linux64.tgz https://scan.coverity.com/download/linux64 --form project=$COVERITY_SCAN_PROJECT_NAME --form token=$COVERITY_SCAN_TOKEN
-    - tar xfz /tmp/cov-analysis-linux64.tgz
-  script:
-    - CI/before_script.linux.sh
-    # Add more than just `openmw` once we can build everything under 3h
-    - cov-analysis-linux64-*/bin/cov-build --dir cov-int cmake --build build -- -j $(nproc) openmw
-  after_script:
-    - tar cfz cov-int.tar.gz cov-int
-    - curl https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
-      --form token=$COVERITY_SCAN_TOKEN --form email=$GITLAB_USER_EMAIL
-      --form file=@cov-int.tar.gz --form version="`git describe --tags`"
-      --form description="`git describe --tags` / $CI_COMMIT_TITLE / $CI_COMMIT_REF_NAME:$CI_PIPELINE_ID"
-  variables:
-    CC: gcc
-    CXX: g++
-  timeout: 8h
-
-Debian_GCC:
-  extends: .Debian
-  cache:
-    key: Debian_GCC.v2
-  before_script:
-    - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-dynamic
-  variables:
-    CC: gcc
-    CXX: g++
-    CCACHE_SIZE: 3G
-  # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
-  timeout: 2h
-
-Debian_GCC_tests:
-  extends: Debian_GCC
-  cache:
-    key: Debian_GCC_tests.v2
-  variables:
-    CCACHE_SIZE: 1G
-    BUILD_TESTS_ONLY: 1
-
-Debian_GCC_Static_Deps:
-  extends: Debian_GCC
-  cache:
-    key: Debian_GCC_Static_Deps
-    paths:
-    - apt-cache/
-    - ccache/
-    - build/extern/fetched/
-  before_script:
-    - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-static
-  variables:
-    CI_OPENMW_USE_STATIC_DEPS: 1
-
-Debian_GCC_Static_Deps_tests:
-  extends: Debian_GCC_Static_Deps
-  cache:
-    key: Debian_GCC_Static_Deps_tests
-  variables:
-    CCACHE_SIZE: 1G
-    BUILD_TESTS_ONLY: 1
-
-Debian_Clang:
-  extends: .Debian
-  before_script:
-    - CI/install_debian_deps.sh clang openmw-deps openmw-deps-dynamic
-  cache:
-    key: Debian_Clang.v2
-  variables:
-    CC: clang
-    CXX: clang++
-    CCACHE_SIZE: 2G
-  # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
-  timeout: 2h
-
-Debian_Clang_tests:
-  extends: Debian_Clang
-  cache:
-    key: Debian_Clang_tests.v2
-  variables:
-    CCACHE_SIZE: 1G
-    BUILD_TESTS_ONLY: 1
-
-.MacOS:
-  image: macos-11-xcode-12
-  tags:
-    - shared-macos-amd64
-  stage: build
-  only:
-    variables:
-      - $CI_PROJECT_ID == "7107382"
-  cache:
-    paths:
-    - ccache/
-  script:
-    - rm -fr build  # remove the build directory
-    - CI/before_install.osx.sh
-    - export CCACHE_BASEDIR="$(pwd)"
-    - export CCACHE_DIR="$(pwd)/ccache"
-    - mkdir -pv "${CCACHE_DIR}"
-    - ccache -z -M "${CCACHE_SIZE}"
-    - CI/before_script.osx.sh
-    - cd build; make -j $(sysctl -n hw.logicalcpu) package
-    - for dmg in *.dmg; do mv "$dmg" "${dmg%.dmg}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}.dmg"; done
-    - ccache -s
-  artifacts:
-    paths:
-      - build/OpenMW-*.dmg
-      - "build/**/*.log"
-
-macOS11_Xcode12:
-  extends: .MacOS
-  image: macos-11-xcode-12
-  allow_failure: true
-  cache:
-    key: macOS11_Xcode12.v1
-  variables:
-    CCACHE_SIZE: 3G
-
-macOS10.15_Xcode11:
-  extends: .MacOS
-  image: macos-10.15-xcode-11
-  cache:
-    key: macOS10.15_Xcode11.v1
-  variables:
-    CCACHE_SIZE: 3G
+# .Debian_Image:
+#   tags:
+#     - docker
+#     - linux
+#   image: debian:bullseye
+
+# .Debian:
+#   extends: .Debian_Image
+#   cache:
+#     paths:
+#     - apt-cache/
+#     - ccache/
+#   stage: build
+#   script:
+#     - export CCACHE_BASEDIR="`pwd`"
+#     - export CCACHE_DIR="`pwd`/ccache" && mkdir -pv "$CCACHE_DIR"
+#     - ccache -z -M "${CCACHE_SIZE}"
+#     - CI/before_script.linux.sh
+#     - cd build
+#     - cmake --build . -- -j $(nproc)
+#     - cmake --install .
+#     - if [[ "${BUILD_TESTS_ONLY}" ]]; then ./openmw_test_suite; fi
+#     - if [[ "${BUILD_TESTS_ONLY}" ]]; then ./openmw_detournavigator_navmeshtilescache_benchmark; fi
+#     - ccache -s
+#   artifacts:
+#     paths:
+#       - build/install/
+
+# Coverity:
+#   extends: .Debian_Image
+#   stage: build
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "schedule"'
+#   before_script:
+#     - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-dynamic coverity
+#     - curl -o /tmp/cov-analysis-linux64.tgz https://scan.coverity.com/download/linux64 --form project=$COVERITY_SCAN_PROJECT_NAME --form token=$COVERITY_SCAN_TOKEN
+#     - tar xfz /tmp/cov-analysis-linux64.tgz
+#   script:
+#     - CI/before_script.linux.sh
+#     # Add more than just `openmw` once we can build everything under 3h
+#     - cov-analysis-linux64-*/bin/cov-build --dir cov-int cmake --build build -- -j $(nproc) openmw
+#   after_script:
+#     - tar cfz cov-int.tar.gz cov-int
+#     - curl https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
+#       --form token=$COVERITY_SCAN_TOKEN --form email=$GITLAB_USER_EMAIL
+#       --form file=@cov-int.tar.gz --form version="`git describe --tags`"
+#       --form description="`git describe --tags` / $CI_COMMIT_TITLE / $CI_COMMIT_REF_NAME:$CI_PIPELINE_ID"
+#   variables:
+#     CC: gcc
+#     CXX: g++
+#   timeout: 8h
+
+# Debian_GCC:
+#   extends: .Debian
+#   cache:
+#     key: Debian_GCC.v2
+#   before_script:
+#     - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-dynamic
+#   variables:
+#     CC: gcc
+#     CXX: g++
+#     CCACHE_SIZE: 3G
+#   # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
+#   timeout: 2h
+
+# Debian_GCC_tests:
+#   extends: Debian_GCC
+#   cache:
+#     key: Debian_GCC_tests.v2
+#   variables:
+#     CCACHE_SIZE: 1G
+#     BUILD_TESTS_ONLY: 1
+
+# Debian_GCC_Static_Deps:
+#   extends: Debian_GCC
+#   cache:
+#     key: Debian_GCC_Static_Deps
+#     paths:
+#     - apt-cache/
+#     - ccache/
+#     - build/extern/fetched/
+#   before_script:
+#     - CI/install_debian_deps.sh gcc openmw-deps openmw-deps-static
+#   variables:
+#     CI_OPENMW_USE_STATIC_DEPS: 1
+
+# Debian_GCC_Static_Deps_tests:
+#   extends: Debian_GCC_Static_Deps
+#   cache:
+#     key: Debian_GCC_Static_Deps_tests
+#   variables:
+#     CCACHE_SIZE: 1G
+#     BUILD_TESTS_ONLY: 1
+
+# Debian_Clang:
+#   extends: .Debian
+#   before_script:
+#     - CI/install_debian_deps.sh clang openmw-deps openmw-deps-dynamic
+#   cache:
+#     key: Debian_Clang.v2
+#   variables:
+#     CC: clang
+#     CXX: clang++
+#     CCACHE_SIZE: 2G
+#   # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
+#   timeout: 2h
+
+# Debian_Clang_tests:
+#   extends: Debian_Clang
+#   cache:
+#     key: Debian_Clang_tests.v2
+#   variables:
+#     CCACHE_SIZE: 1G
+#     BUILD_TESTS_ONLY: 1
+
+# .MacOS:
+#   image: macos-11-xcode-12
+#   tags:
+#     - shared-macos-amd64
+#   stage: build
+#   only:
+#     variables:
+#       - $CI_PROJECT_ID == "7107382"
+#   cache:
+#     paths:
+#     - ccache/
+#   script:
+#     - rm -fr build  # remove the build directory
+#     - CI/before_install.osx.sh
+#     - export CCACHE_BASEDIR="$(pwd)"
+#     - export CCACHE_DIR="$(pwd)/ccache"
+#     - mkdir -pv "${CCACHE_DIR}"
+#     - ccache -z -M "${CCACHE_SIZE}"
+#     - CI/before_script.osx.sh
+#     - cd build; make -j $(sysctl -n hw.logicalcpu) package
+#     - for dmg in *.dmg; do mv "$dmg" "${dmg%.dmg}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}.dmg"; done
+#     - ccache -s
+#   artifacts:
+#     paths:
+#       - build/OpenMW-*.dmg
+#       - "build/**/*.log"
+
+# macOS11_Xcode12:
+#   extends: .MacOS
+#   image: macos-11-xcode-12
+#   allow_failure: true
+#   cache:
+#     key: macOS11_Xcode12.v1
+#   variables:
+#     CCACHE_SIZE: 3G
+
+# macOS10.15_Xcode11:
+#   extends: .MacOS
+#   image: macos-10.15-xcode-11
+#   cache:
+#     key: macOS10.15_Xcode11.v1
+#   variables:
+#     CCACHE_SIZE: 3G
 
 variables: &engine-targets
   targets: "openmw,openmw-essimporter,openmw-iniimporter,openmw-launcher,openmw-wizard"
@@ -186,6 +186,7 @@ variables: &tests-targets
   - choco install vswhere -y
   - choco install ninja -y
   - choco install python -y
+  - choco install windows-sdk-10-version-2004-windbg -y
   - refreshenv
   stage: build
   script:
@@ -205,8 +206,13 @@ variables: &tests-targets
       }
     - 7z a -tzip ..\..\OpenMW_MSVC2019_64_${package}_${config}_${CI_COMMIT_REF_NAME}.zip '*'
     - if ($executables) { foreach ($exe in $executables.Split(',')) { & .\$exe } }
+    - $env:Path += ";C:\Program Files (x86)\Windows Kits\10\Debuggers\x64"
+    - cdb.exe -g -G .\openmw_test_suite.exe
+    - cdb.exe -g -G .\openmw_detournavigator_navmeshtilescache_benchmark.exe
   after_script:
     - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
+    - pwd
+    - dir
   cache:
     key: ninja-v2
     paths:
@@ -226,54 +232,54 @@ variables: &tests-targets
     - MSVC2019_64_Ninja/*/*/*/*/*/*/*.log
     - MSVC2019_64_Ninja/*/*/*/*/*/*/*/*.log
 
-Windows_Ninja_Engine_Release:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *engine-targets
-    config: "Release"
-
-Windows_Ninja_Engine_Debug:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *engine-targets
-    config: "Debug"
-
-Windows_Ninja_Engine_RelWithDebInfo:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *engine-targets
-    config: "RelWithDebInfo"
-
-Windows_Ninja_CS_Release:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *cs-targets
-    config: "Release"
-
-Windows_Ninja_CS_Debug:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *cs-targets
-    config: "Debug"
-
-Windows_Ninja_CS_RelWithDebInfo:
-  extends:
-    - .Windows_Ninja_Base
-  variables:
-    <<: *cs-targets
-    config: "RelWithDebInfo"
+# Windows_Ninja_Engine_Release:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "Release"
+
+# Windows_Ninja_Engine_Debug:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "Debug"
+
+# Windows_Ninja_Engine_RelWithDebInfo:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "RelWithDebInfo"
+
+# Windows_Ninja_CS_Release:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "Release"
+
+# Windows_Ninja_CS_Debug:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "Debug"
+
+# Windows_Ninja_CS_RelWithDebInfo:
+#   extends:
+#     - .Windows_Ninja_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "RelWithDebInfo"
 
 Windows_Ninja_Tests_RelWithDebInfo:
   extends: .Windows_Ninja_Base
   stage: build
   variables:
     <<: *tests-targets
-    config: "RelWithDebInfo"
+    config: "Debug"
     # Gitlab can't successfully execute following binaries due to unknown reason
     # executables: "openmw_test_suite.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe"
 
@@ -288,6 +294,7 @@ Windows_Ninja_Tests_RelWithDebInfo:
   - choco install cmake.install --installargs 'ADD_CMAKE_TO_PATH=System' -y
   - choco install vswhere -y
   - choco install python -y
+  - choco install windows-sdk-10-version-2004-windbg -y
   - refreshenv
   stage: build
   script:
@@ -306,8 +313,13 @@ Windows_Ninja_Tests_RelWithDebInfo:
       }
     - 7z a -tzip ..\..\OpenMW_MSVC2019_64_${package}_${config}_${CI_COMMIT_REF_NAME}.zip '*'
     - if ($executables) { foreach ($exe in $executables.Split(',')) { & .\$exe } }
+    - $env:Path += ";C:\Program Files (x86)\Windows Kits\10\Debuggers\x64"
+    - cdb.exe -g -G .\openmw_test_suite.exe
+    - cdb.exe -g -G .\openmw_detournavigator_navmeshtilescache_benchmark.exe
   after_script:
     - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
+    - pwd
+    - dir
   cache:
     key: msbuild-v2
     paths:
@@ -327,88 +339,89 @@ Windows_Ninja_Tests_RelWithDebInfo:
     - MSVC2019_64/*/*/*/*/*/*/*.log
     - MSVC2019_64/*/*/*/*/*/*/*/*.log
 
-Windows_MSBuild_Engine_Release:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *engine-targets
-    config: "Release"
-
-Windows_MSBuild_Engine_Debug:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *engine-targets
-    config: "Debug"
-
-Windows_MSBuild_Engine_RelWithDebInfo:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *engine-targets
-    config: "RelWithDebInfo"
-
-Windows_MSBuild_CS_Release:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *cs-targets
-    config: "Release"
-
-Windows_MSBuild_CS_Debug:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *cs-targets
-    config: "Debug"
-
-Windows_MSBuild_CS_RelWithDebInfo:
-  extends:
-    - .Windows_MSBuild_Base
-  variables:
-    <<: *cs-targets
-    config: "RelWithDebInfo"
+# Windows_MSBuild_Engine_Release:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "Release"
+
+# Windows_MSBuild_Engine_Debug:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "Debug"
+
+# Windows_MSBuild_Engine_RelWithDebInfo:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *engine-targets
+#     config: "RelWithDebInfo"
+
+# Windows_MSBuild_CS_Release:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "Release"
+
+# Windows_MSBuild_CS_Debug:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "Debug"
+
+# Windows_MSBuild_CS_RelWithDebInfo:
+#   extends:
+#     - .Windows_MSBuild_Base
+#   variables:
+#     <<: *cs-targets
+#     config: "RelWithDebInfo"
 
 Windows_MSBuild_Tests_RelWithDebInfo:
   extends: .Windows_MSBuild_Base
   stage: build
   variables:
     <<: *tests-targets
-    config: "RelWithDebInfo"
+    config: "Debug"
     # Gitlab can't successfully execute following binaries due to unknown reason
     # executables: "openmw_test_suite.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe"
-
-Debian_AndroidNDK_arm64-v8a:
-  tags:
-    - linux
-  image: debian:bullseye
-  variables:
-    CCACHE_SIZE: 3G
-  cache:
-    key: Debian_AndroidNDK_arm64-v8a.v3
-    paths:
-      - apt-cache/
-      - ccache/
-      - build/extern/fetched/
-  before_script:
-    - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
-    - echo "deb http://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list
-    - echo "google-android-ndk-installer google-android-installers/mirror select https://dl.google.com" | debconf-set-selections
-    - apt-get update -yq
-    - apt-get -q -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake ccache curl unzip git build-essential google-android-ndk-installer
-  stage: build
-  script:
-    - export CCACHE_BASEDIR="`pwd`"
-    - export CCACHE_DIR="`pwd`/ccache" && mkdir -pv "$CCACHE_DIR"
-    - ccache -z -M "${CCACHE_SIZE}"
-    - CI/before_install.android.sh
-    - CI/before_script.android.sh
-    - cd build
-    - cmake --build . -- -j $(nproc)
-    - cmake --install .
-    - ccache -s
-  artifacts:
-    paths:
-      - build/install/
-  # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
-  timeout: 1h30m
+    executables: "openmw_test_suite.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe"
+
+# Debian_AndroidNDK_arm64-v8a:
+#   tags:
+#     - linux
+#   image: debian:bullseye
+#   variables:
+#     CCACHE_SIZE: 3G
+#   cache:
+#     key: Debian_AndroidNDK_arm64-v8a.v3
+#     paths:
+#       - apt-cache/
+#       - ccache/
+#       - build/extern/fetched/
+#   before_script:
+#     - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
+#     - echo "deb http://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list
+#     - echo "google-android-ndk-installer google-android-installers/mirror select https://dl.google.com" | debconf-set-selections
+#     - apt-get update -yq
+#     - apt-get -q -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake ccache curl unzip git build-essential google-android-ndk-installer
+#   stage: build
+#   script:
+#     - export CCACHE_BASEDIR="`pwd`"
+#     - export CCACHE_DIR="`pwd`/ccache" && mkdir -pv "$CCACHE_DIR"
+#     - ccache -z -M "${CCACHE_SIZE}"
+#     - CI/before_install.android.sh
+#     - CI/before_script.android.sh
+#     - cd build
+#     - cmake --build . -- -j $(nproc)
+#     - cmake --install .
+#     - ccache -s
+#   artifacts:
+#     paths:
+#       - build/install/
+#   # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
+#   timeout: 1h30m