diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7f8c42b1fc62d94012e370d6aaf4dac997043cf4..309fad5cf9c1f46a3f3c20b343a3c4a4f6c736af 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ test_single:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
image: "python:$PYTHON_VERSION"
services:
- - name: postgis/postgis:16-3.4
+ - name: timescale/timescaledb-ha:pg17
alias: postgres
command: ["postgres", "-c", "fsync=off"]
before_script:
@@ -44,7 +44,7 @@ test_single:
- $WEATHERDB_TEST_ARTIFACT_DIR/*
interruptible: true
variables:
- PYTHON_VERSION: 3.8
+ PYTHON_VERSION: 3.12
PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
@@ -77,7 +77,7 @@ test_matrix:
extends: test_single
parallel:
matrix:
- - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+ - PY_VERSION: ["3.8", "3.9","3.10","3.11","3.13"]
variables:
PYTHON_VERSION: $PY_VERSION
@@ -92,8 +92,7 @@ test_download_rasters:
variables:
PYTHON_VERSION: $PY_VERSION
parallel:
- matrix:
- - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+ matrix: !reference [test_matrix, parallel, matrix ]
build:
stage: build
@@ -101,7 +100,7 @@ build:
- if: $CI_COMMIT_TAG =~ /^[Vv]*\d+\.\d+\.\d+$/
dependencies:
- test_matrix
- image: python:3.11
+ image: python:3.12
script:
- python -m pip install --upgrade pip --root-user-action=ignore
- python -m pip install setuptools wheel build setuptools-scm --root-user-action=ignore
diff --git a/CHANGES.md b/CHANGES.md
index 29f6c89dc89adb8cb13117f720563f7be178deec..e4257d780ec7e2ee918b15e61e017fd1b5b8a21d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,11 @@
# Change-log
+## Version 1.2.3
+
+- fix: dropping stations
+- fix: test environment as DWD kicked station 2388
+- docker: optimize image with timescaledb image instead of postgis
+
## Version 1.2.2
- config: define utf-8 as default encoding for config files
@@ -7,7 +13,7 @@
## Version 1.2.1
-- fix: filllup linear regression did not take configuration value and therefor did never work
+- fix: fillup linear regression did not take configuration value and therefor did never work
- config: change LINEAR_INTERPOLATION_LIMIT to be a category
- cli: add version parameter
- multiprocessing: make process count configurable through user configuration
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index ae92d49f535dc285e1b9f67f7f3349e171ef11f1..2d99a74d9d112c87c5d284eaeabe8c907551038c 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -38,11 +38,13 @@ services:
target: /src/tests
db:
- image: postgis/postgis:16-3.4
+ image: timescale/timescaledb-ha:pg17 # about 1 minute faster for tests
+ # image: postgis/postgis:17-3.4
environment:
POSTGRES_DB: weatherdb
POSTGRES_USER: postgres
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
+ shm_size: '2g'
ports:
- 5454:5432
volumes:
diff --git a/tests/baseTest.py b/tests/baseTest.py
index 9ec0dd301af0f9ea1e8ac1956462821cecfd2a10..fa1c42ab4dd3ffa1916497c42140f2c15e1b202a 100644
--- a/tests/baseTest.py
+++ b/tests/baseTest.py
@@ -27,11 +27,12 @@ else:
class BaseTestCases(unittest.TestCase):
broker = wdb.broker.Broker()
db_engine = wdb.db.connections.db_engine
- test_stids = [1224, 1443, 2388, 7243, 1346, 684, 757]
+ test_stids = [1224, 1443, 7243, 1346, 684, 757]
log = wdb.utils.logging.log
- @staticmethod
- def empty_db():
+ @classmethod
+ def empty_db(cls):
+ cls.log.debug("Dropping all tables and schema from test database.")
with wdb.db.connections.db_engine.connect() as conn:
models.ModelBase.metadata.drop_all(conn)
conn.execute(sa.schema.DropSchema("timeseries", cascade=True, if_exists=True))
diff --git a/tests/test_initDB.py b/tests/test_initDB.py
index 39027e355f4631df485c4db42135f3f277126d17..e2f4cd41af56b11879998bfb1c236f520e616f70 100644
--- a/tests/test_initDB.py
+++ b/tests/test_initDB.py
@@ -7,6 +7,7 @@ from pathlib import Path
import pandas as pd
from distutils.util import strtobool
from datetime import timezone
+from datetime import datetime
import shutil
from tempfile import TemporaryDirectory
@@ -252,7 +253,7 @@ class InitDBTestCases(BaseTestCases):
def check_update_meta(self):
with self.db_engine.connect() as conn:
for model, n_expct in zip([models.MetaP, models.MetaT, models.MetaPD, models.MetaET],
- [7, 5, 7, 3]):
+ [6, 4, 6, 3]):
# check number of stations
with self.subTest(model=model):
stmnt = sa.select(sa.func.count('*')).select_from(model)
@@ -503,6 +504,7 @@ class InitDBTestCases(BaseTestCases):
STEPS = ["update_meta", "update_raw", "update_ma_raster",
"quality_check", "fillup", "update_richter_class", "richter_correct",
"vacuum", "create_ts"]
+
# get steps from cli arguments
steps = cliargs.steps
if steps == "all":
@@ -527,14 +529,17 @@ class InitDBTestCases(BaseTestCases):
steps = STEPS[STEPS.index(highest_step)+1:STEPS.index(steps[-1])+1]
# run steps
+ times = dict()
for step in steps:
# run step
try:
if not cliargs.just_check:
+ start_time = datetime.now()
getattr(self, f"step_{step}")(
stids=self.test_stids,
skip_missing_stids=True
)
+ times[step] = datetime.now() - start_time
except Exception as e:
self.log.exception(e)
self.fail("{} failed ({}: {})".format(step, type(e), e))
@@ -554,6 +559,12 @@ class InitDBTestCases(BaseTestCases):
self.log.debug(f"Step {step} failed. Highest run step is still {self.broker.get_setting('highest_run_step')}")
break
+ # print times
+ self.log.info(
+ "Times for steps:\n" +
+ "\n".join([f"{step}: {dt.seconds//60}:{dt.seconds-(dt.seconds//60*60):0>2}"
+ for step, dt in times.items()]))
+
# cli entry point
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
diff --git a/weatherdb/station/StationBases.py b/weatherdb/station/StationBases.py
index bbd85dd01e6c4b606206a96b4bcbc2745bb3060d..9d7c6aad54dd13ebb2feec2f3eb5d546f4e34015 100644
--- a/weatherdb/station/StationBases.py
+++ b/weatherdb/station/StationBases.py
@@ -595,7 +595,7 @@ class StationBase:
DROP TABLE IF EXISTS timeseries."{self.id}_{self._para}";
DELETE FROM meta_{self._para} WHERE station_id={self.id};
DELETE FROM station_ma_raster WHERE station_id={self.id} and parameter='{self._para}';
- DELETE FROM station_ma_timeseries WHERE station_id={self.id} and parameter='{self._para}';
+ DELETE FROM station_ma_timeserie WHERE station_id={self.id} and parameter='{self._para}';
INSERT INTO dropped_stations(station_id, parameter, why, timestamp)
VALUES ('{self.id}', '{self._para}', '{why}', NOW())
ON CONFLICT (station_id, parameter)
diff --git a/weatherdb/station/StationP.py b/weatherdb/station/StationP.py
index 7bcd37b58eb886065c77fc3dc25a2b3006f06543..ce6f9110838562d0cdb99610a8faabf33d3310ba 100644
--- a/weatherdb/station/StationP.py
+++ b/weatherdb/station/StationP.py
@@ -240,8 +240,8 @@ class StationP(StationPBase):
@db_engine.deco_create_privilege
def _create_timeseries_table(self):
"""Create the timeseries table in the DB if it is not yet existing."""
- sql_add_table = '''
- CREATE TABLE IF NOT EXISTS timeseries."{stid}_{para}" (
+ sql_add_table = f'''
+ CREATE TABLE IF NOT EXISTS timeseries."{self.id}_{self._para}" (
timestamp timestamp PRIMARY KEY,
raw int4,
qn smallint,
@@ -250,7 +250,7 @@ class StationP(StationPBase):
filled_by int2,
corr int4
);
- '''.format(stid=self.id, para=self._para)
+ '''
with db_engine.connect() as con:
con.execute(sqltxt(sql_add_table))
con.commit()