Run integration tests in CIci_integration_tests

6 files changed, 99 insertions, 9 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 667f007660..5045bb3adb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,8 +4,8 @@ include:
 # Note: We set `needs` on each job to control the job DAG.
 # See https://docs.gitlab.com/ee/ci/yaml/#needs
 stages:
-  - test
   - build
+  - test
 
 # https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
 variables:
@@ -16,6 +16,7 @@ variables:
   CACHE_COMPRESSION_LEVEL: "fast"
   SAST_EXCLUDED_ANALYZERS: "bandit"
   SAST_EXCLUDED_PATHS: "extern"
+  GIT_DEPTH: 1
 
 .Ubuntu_Image:
   tags:
@@ -23,8 +24,7 @@ variables:
     - linux
   image: ubuntu:focal
   rules:
-    - if: $CI_PIPELINE_SOURCE == "push"  
-  
+    - if: $CI_PIPELINE_SOURCE == "push"
 
 .Ubuntu:
   extends: .Ubuntu_Image
@@ -220,6 +220,24 @@ Ubuntu_GCC_tests_coverage:
         path: coverage.xml
       junit: build/tests.xml
 
+Ubuntu_GCC_integration_tests:
+  extends: .Ubuntu_Image
+  stage: test
+  needs:
+    - Ubuntu_GCC
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  cache:
+    key: Ubuntu_GCC_integration_tests.v1
+    paths:
+      - .cache/pip
+      - apt-cache/
+  before_script:
+    - CI/install_debian_deps.sh openmw-integration-tests
+    - pip3 install --user numpy matplotlib termtables click
+  script:
+    - CI/run_integration_tests.sh
+
 Ubuntu_Static_Deps:
   extends: Ubuntu_Clang
   rules:
@@ -637,7 +655,7 @@ Ubuntu_AndroidNDK_arm64-v8a:
     - linux
   image: psi29a/android-ndk:focal-ndk22
   rules:
-    - if: $CI_PIPELINE_SOURCE == "push"  
+    - if: $CI_PIPELINE_SOURCE == "push"
   variables:
     CCACHE_SIZE: 3G
   cache:
diff --git a/CI/before_script.linux.sh b/CI/before_script.linux.sh
index 41d271fc70..026de1ef9b 100755
--- a/CI/before_script.linux.sh
+++ b/CI/before_script.linux.sh
@@ -25,7 +25,6 @@ declare -a CMAKE_CONF_OPTS=(
     -DBUILD_SHARED_LIBS=OFF
     -DUSE_SYSTEM_TINYXML=ON
     -DOPENMW_USE_SYSTEM_RECASTNAVIGATION=ON
-    -DCMAKE_INSTALL_PREFIX=install
     -DOPENMW_CXX_FLAGS="-Werror -Werror=implicit-fallthrough"  # flags specific to OpenMW project
 )
 
@@ -100,6 +99,17 @@ if [[ "${BUILD_TESTS_ONLY}" ]]; then
         -DGTEST_ROOT="${GOOGLETEST_DIR}" \
         -DGMOCK_ROOT="${GOOGLETEST_DIR}" \
         ..
+elif [[ ${RUN_INTEGRATION_TESTS} ]]; then
+    ${ANALYZE} cmake \
+        "${CMAKE_CONF_OPTS[@]}" \
+        -DBUILD_BSATOOL=OFF \
+        -DBUILD_ESMTOOL=OFF \
+        -DBUILD_LAUNCHER=OFF \
+        -DBUILD_MWINIIMPORTER=OFF \
+        -DBUILD_ESSIMPORTER=OFF \
+        -DBUILD_OPENCS=OFF \
+        -DBUILD_WIZARD=OFF \
+        ..
 else
     ${ANALYZE} cmake \
         "${CMAKE_CONF_OPTS[@]}" \
diff --git a/CI/install_debian_deps.sh b/CI/install_debian_deps.sh
index c089558208..8342d89722 100755
--- a/CI/install_debian_deps.sh
+++ b/CI/install_debian_deps.sh
@@ -42,6 +42,39 @@ declare -rA GROUPED_DEPS=(
   "
 
   [openmw-coverage]="gcovr"
+
+  [openmw-integration-tests]="
+    ca-certificates
+    git
+    git-lfs
+    libavcodec58
+    libavformat58
+    libavutil56
+    libboost-filesystem1.71.0
+    libboost-iostreams1.71.0
+    libboost-program-options1.71.0
+    libboost-system1.71.0
+    libbullet2.88
+    libcollada-dom2.4-dp0
+    libicu66
+    libjpeg8
+    libluajit-5.1-2
+    liblz4-1
+    libmyguiengine3debian1v5
+    libopenal1
+    libopenscenegraph161
+    libpng16-16
+    libqt5opengl5
+    librecast1
+    libsdl2-2.0-0
+    libsqlite3-0
+    libswresample3
+    libswscale5
+    libtinyxml2.6.2v5
+    libyaml-cpp0.6
+    python3-pip
+    xvfb
+  "
 )
 
 if [[ $# -eq 0 ]]; then
diff --git a/CI/run_integration_tests.sh b/CI/run_integration_tests.sh
new file mode 100755
index 0000000000..55eb8d40e3
--- /dev/null
+++ b/CI/run_integration_tests.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -ex
+
+find .
+
+git clone --depth=1 https://gitlab.com/OpenMW/example-suite.git
+
+xvfb-run --auto-servernum --server-args='-screen 0 640x480x24x60' \
+    scripts/integration_tests.py --omw build/install/bin/openmw --workdir integration_tests_output example-suite/
+
+ls integration_tests_output/*.osg_stats.log | while read v; do
+    echo "${v}"
+    scripts/osg_stats.py --stats '.*' --regexp_match < "${v}"
+done
diff --git a/scripts/integration_tests.py b/scripts/integration_tests.py
index b125e688fb..00c6ec80ce 100755
--- a/scripts/integration_tests.py
+++ b/scripts/integration_tests.py
@@ -51,14 +51,28 @@ def runTest(name):
         )
         if (test_dir / "test.omwscripts").exists():
             omw_cfg.write("content=test.omwscripts\n")
+    with open(config_dir / "settings.cfg", "a", encoding="utf-8") as settings_cfg:
+        settings_cfg.write(
+            "[Video]\n"
+            "resolution x = 640\n"
+            "resolution y = 480\n"
+            "framerate limit = 60\n"
+        )
+    stdout_lines = list()
     with subprocess.Popen(
-        [f"{openmw_binary}", "--replace=config", f"--config={config_dir}", "--skip-menu", "--no-grab"],
+        [openmw_binary, "--replace=config", f"--config={config_dir}", "--skip-menu", "--no-grab"],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         encoding="utf-8",
+        env={
+            "OPENMW_OSG_STATS_FILE": work_dir / f"{name}.{time_str}.osg_stats.log",
+            "OPENMW_OSG_STATS_LIST": "times",
+            **os.environ,
+        },
     ) as process:
         quit_requested = False
         for line in process.stdout:
+            stdout_lines.append(line)
             words = line.split(" ")
             if len(words) > 1 and words[1] == "E]":
                 print(line, end="")
@@ -75,7 +89,10 @@ def runTest(name):
         process.wait(5)
         if not quit_requested:
             print("ERROR: Unexpected termination")
-    shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log")
+    if os.path.exists(config_dir / "openmw.log"):
+        shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log")
+    else:
+        sys.stdout.writelines(stdout_lines)
     print(f"{name} finished")
 
 
@@ -84,4 +101,3 @@ for entry in tests_dir.glob("test_*"):
         runTest(entry.name)
 shutil.rmtree(config_dir, ignore_errors=True)
 shutil.rmtree(userdata_dir, ignore_errors=True)
-
diff --git a/scripts/osg_stats.py b/scripts/osg_stats.py
index cc4314cd04..3a5067851d 100755
--- a/scripts/osg_stats.py
+++ b/scripts/osg_stats.py
@@ -76,7 +76,7 @@ def main(print_keys, regexp_match, timeseries, hist, hist_ratio, stdev_hist, plo
     def matching_keys(patterns):
         if regexp_match:
             return [key for pattern in patterns for key in keys if re.search(pattern, key)]
-        return keys
+        return patterns
     if timeseries:
         draw_timeseries(sources=frames, keys=matching_keys(timeseries), add_sum=timeseries_sum,
                         begin_frame=begin_frame, end_frame=end_frame)