From e76eaaa66431b3f8606789ea1b8fde79f2572425 Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 09:02:46 +0100
Subject: [PATCH 1/8] cli: add version parameter
---
weatherdb/cli.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/weatherdb/cli.py b/weatherdb/cli.py
index 5ba77f1..7c8979b 100644
--- a/weatherdb/cli.py
+++ b/weatherdb/cli.py
@@ -16,6 +16,7 @@ import weatherdb
show_default=True
)
)
+@click.version_option(weatherdb.__version__)
@click.option('--do-logging/--no-logging',
is_flag=True, default=True, show_default=True,
help="Should the logging be done to the console?")
--
GitLab
From d75e8a12be271c963ee935c74a0d3ce6b810fba3 Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 09:20:22 +0100
Subject: [PATCH 2/8] fix: filllup linear regression did not take configuration
value and therefor did never work
---
CHANGES.md | 13 ++++++++++++-
weatherdb/station/StationBases.py | 2 +-
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index e3d520e..d0c663a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,15 @@
# Change-log
+## Version 1.2.1
+
+- fix: filllup linear regression did not take configuration value and therefor did never work
+- cli: add version parameter
+
## Version 1.2.0
+> [!WARNING]
+> This versions update did have an error and the lienear regression doesn't work at all. PLease use version 1.2.1
+
- tests: add test rasters and make downloading the rasters a manual job on GitLab as there were many problems
- fillup: add method to linearly interpolate residual missing values
@@ -17,7 +25,10 @@
## Version 1.1.0
-- !!rename module from weatherDB to weatherdb to be compliant with PEP 8!!
+> [!NOTE]
+> **major change:** The module got renamed to `weatherdb`, you need to adapt this in your import and cli statements
+
+- rename module from weatherDB to weatherdb to be compliant with PEP 8
So you need to import the module now with `import weatherdb`
- fix some missing stationN variables to stationsP
- fix: TimestampPeriod._check_period
diff --git a/weatherdb/station/StationBases.py b/weatherdb/station/StationBases.py
index 3b20f61..9fa672f 100644
--- a/weatherdb/station/StationBases.py
+++ b/weatherdb/station/StationBases.py
@@ -1494,7 +1494,7 @@ class StationBase:
# linear interpolation for the last missing values
lr_limit = config.get(
"weatherdb",
- "LINEAR_INTERPOLATION_LIMIT_{self._para_base}",
+ f"LINEAR_INTERPOLATION_LIMIT_{self._para_base}",
fallback="0")
if lr_limit != "0":
sql_format_dict.update(dict(
--
GitLab
From f9bf044ce0624479cb64fc65d2579f6b919986af Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 09:45:00 +0100
Subject: [PATCH 3/8] gitlab-ci: add gitlab release
---
.gitlab-ci.yml | 27 +++++++++++++++++++++------
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48cec8b..a740d1a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,7 +3,7 @@ stages:
- test_matrix
- test_manual
- build
- - deploy
+ - release
workflow:
auto_cancel:
@@ -97,11 +97,11 @@ test_download_rasters:
- PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
build:
+ stage: build
rules:
- - if: $CI_COMMIT_TAG =~ /^[Vv]\d+\.\d+\.\d+$/
+ - if: $CI_COMMIT_TAG =~ /^[Vv]*\d+\.\d+\.\d+$/
dependencies:
- test_matrix
- stage: build
image: python:3.11
script:
- python -m pip install --upgrade pip --root-user-action=ignore
@@ -115,12 +115,12 @@ build:
tags:
- docker
-deploy:
+release_pypi:
+ stage: release
rules:
- !reference [build, rules]
dependencies:
- build
- stage: deploy
image: python:3.11
script:
- python -m pip install --upgrade pip --root-user-action=ignore
@@ -131,4 +131,19 @@ deploy:
TWINE_PASSWORD: $TWINE_PASSWORD
tags:
- docker
- interruptible: false
\ No newline at end of file
+ interruptible: false
+
+release_gitlab:
+ stage: release
+ rules:
+ - !reference [build, rules]
+ image: registry.gitlab.com/gitlab-org/release-cli:latest
+ dependencies:
+ - build
+ - release_pypi
+ script:
+ - echo "running release_gitlab"
+ release:
+ name: '$CI_COMMIT_TAG'
+ tag_name: '$CI_COMMIT_TAG'
+ description: '$CI_COMMIT_TAG_MESSAGE'
\ No newline at end of file
--
GitLab
From 84cf5caa8abe7292021a293c1e9db78d1c6b612c Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 11:55:44 +0100
Subject: [PATCH 4/8] fillup: linear reg, add explicit type casts
---
weatherdb/station/StationBases.py | 29 ++++++++++++++++++-----------
1 file changed, 18 insertions(+), 11 deletions(-)
diff --git a/weatherdb/station/StationBases.py b/weatherdb/station/StationBases.py
index 9fa672f..30dcb1b 100644
--- a/weatherdb/station/StationBases.py
+++ b/weatherdb/station/StationBases.py
@@ -1507,8 +1507,9 @@ class StationBase:
WITH empty_periods AS (
SELECT *
FROM ( SELECT
- CASE WHEN dist_next>'{self._interval}'::interval
- THEN timestamp ELSE NULL
+ CASE WHEN dist_next::interval>'{self._interval}'::interval
+ THEN timestamp
+ ELSE NULL
END AS start,
CASE WHEN dist_next>'{self._interval}'::interval
THEN LEAD(timestamp) OVER (ORDER BY timestamp)
@@ -1516,8 +1517,12 @@ class StationBase:
END AS end
FROM (
SELECT *,
- timestamp - lag(timestamp, 1) OVER ( ORDER BY timestamp) AS dist_prev,
- lead(timestamp, 1) OVER ( ORDER BY timestamp) - timestamp AS dist_next
+ (timestamp::timestamp
+ - lag(timestamp::timestamp, 1) OVER ( ORDER BY timestamp)
+ )::interval AS dist_prev,
+ (lead(timestamp::timestamp, 1) OVER ( ORDER BY timestamp)
+ - timestamp::timestamp
+ )::interval AS dist_next
FROM new_filled_{self.id}_{self._para}
WHERE filled IS NOT NULL) t
WHERE t.dist_prev > '{self._interval}'::interval
@@ -1529,18 +1534,20 @@ class StationBase:
ttss.filled AS value_start,
ep.end AS timestamp_end,
ttse.filled AS value_end,
- (ttse.filled - ttss.filled)::numeric/(EXTRACT(EPOCH FROM (ep.end - ep.start))/EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric as slope
+ (ttse.filled - ttss.filled)::numeric/(EXTRACT(EPOCH FROM (ep.end::timestamp - ep.start::timestamp))/EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric as slope
FROM empty_periods ep
- LEFT JOIN new_filled_{self.id}_{self._para} ttss ON ep.start = ttss.timestamp
- LEFT JOIN new_filled_{self.id}_{self._para} ttse ON ep.end = ttse.timestamp
- where (ep.end - ep.start - '{self._interval}'::interval) <= '{lr_limit}'::interval
+ LEFT JOIN new_filled_{self.id}_{self._para} ttss ON ep.start::timestamp = ttss.timestamp::timestamp
+ LEFT JOIN new_filled_{self.id}_{self._para} ttse ON ep.end::timestamp = ttse.timestamp::timestamp
+ WHERE (ep.end::timestamp - ep.start::timestamp - '{self._interval}'::interval) <= '{lr_limit}'::interval
loop
execute FORMAT(
$$
UPDATE new_filled_{self.id}_{self._para} ts
- SET filled=%2$L + (EXTRACT(EPOCH FROM ts.timestamp - %1$L)::numeric/(EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric * %5$L),
- filled_by=-1
- WHERE ts.timestamp>%1$L and ts.timestamp<%3$L;
+ SET filled=%2$L::numeric + (%5$L::numeric *
+ EXTRACT(EPOCH FROM ts.timestamp::timestamp - %1$L::timestamp)::numeric
+ /EXTRACT(EPOCH FROM '{self._interval}'::interval)::numeric),
+ filled_by={"ARRAY[-1]::smallint[]" if self._filled_by_n>1 else "-1::smallint"}
+ WHERE ts.timestamp > %1$L AND ts.timestamp < %3$L;
$$,
reg_borders.timestamp_start,
reg_borders.value_start,
--
GitLab
From 411e0ef6c6b47d2c0d5acdc2a459e6892c0512c7 Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 12:32:57 +0100
Subject: [PATCH 5/8] multiprocessing: make process count configurable through
user configuration
---
CHANGES.md | 1 +
weatherdb/config/config_default.ini | 15 +++++++++++++++
weatherdb/stations/StationsBase.py | 15 ++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index d0c663a..45ee17d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -4,6 +4,7 @@
- fix: filllup linear regression did not take configuration value and therefor did never work
- cli: add version parameter
+- multiprocessing: make process count configurable through user configuration
## Version 1.2.0
diff --git a/weatherdb/config/config_default.ini b/weatherdb/config/config_default.ini
index 7fe86e1..f0d9131 100644
--- a/weatherdb/config/config_default.ini
+++ b/weatherdb/config/config_default.ini
@@ -167,6 +167,21 @@ T = 150000
; Evapotranspiration (ET)
ET = 150000
+
+[system:parallel_processes]
+; You can configure how many processes are getting started in parrallel when tghe database is updated.
+; depending on your system, the database can handle more or less processes
+; There are 2 possible ways to configure this.
+; either define the absolute number of processes:
+; ABSOLUTE =
+
+; or define the number of processes in relation to the number of CPUs,
+; by defining how many CPUs should be used less than the maximum available CPUs
+; if the absolute number is defined, the relative number is ignored
+; This is the default setting of cpu count -1
+CPU_COUNT_MINUS = 1
+
+
; for developpers:
; ----------------
;
diff --git a/weatherdb/stations/StationsBase.py b/weatherdb/stations/StationsBase.py
index 2feb7a2..906b31f 100644
--- a/weatherdb/stations/StationsBase.py
+++ b/weatherdb/stations/StationsBase.py
@@ -20,6 +20,7 @@ from ..utils.dwd import get_dwd_meta, get_cdc_file_list
from ..station.StationBases import StationBase
from ..db import models
from ..db.queries.get_quotient import _get_quotient
+from ..config import config
# set settings
# ############
@@ -567,7 +568,16 @@ class StationsBase:
return pbar
def _run_method(self, stations, method, name, kwds=dict(),
- do_mp=True, processes=mp.cpu_count()-1, **kwargs):
+ do_mp=True,
+ processes=config.getint(
+ "system:parallel_processes",
+ "absolute",
+ fallback=mp.cpu_count() - \
+ config.getint(
+ "system:parallel_processes",
+ "cpu_count_minus",
+ fallback=1)),
+ **kwargs):
"""Run methods of the given stations objects in multiprocessing/threading mode.
Parameters
@@ -589,6 +599,9 @@ class StationsBase:
processes : int, optional
The number of processes that should get started simultaneously.
If 1 or less, then the process is computed as a simple loop, so there is no multiprocessing or threading done.
+ There are two possible configuration values in the config file to set the number of processes:
+ - system:parallel_processes:absolute: The absolute number of processes to start.
+ - system:parallel_processes:cpu_count_minus: The number of CPU kernels are substracted by this number, to get the number of processes to start.
The default is the cpu count -1.
"""
log.info(
--
GitLab
From 94df752c7fed1c7995c98c5f498d99f39a86b7d3 Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 12:38:50 +0100
Subject: [PATCH 6/8] broker: prevent broker inactivation by other broker
---
CHANGES.md | 1 +
weatherdb/broker.py | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index 45ee17d..4f62f73 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -5,6 +5,7 @@
- fix: filllup linear regression did not take configuration value and therefor did never work
- cli: add version parameter
- multiprocessing: make process count configurable through user configuration
+- broker: prevent broker inactivation by other broker
## Version 1.2.0
diff --git a/weatherdb/broker.py b/weatherdb/broker.py
index c7eac15..a0f55c1 100644
--- a/weatherdb/broker.py
+++ b/weatherdb/broker.py
@@ -647,8 +647,9 @@ class Broker(object):
is_active : bool
Whether the broker is active.
"""
+ if self._is_active != is_active:
+ self.set_setting("broker_active", str(is_active))
self._is_active = is_active
- self.set_setting("broker_active", str(is_active))
def _deactivate(self):
self.is_active = False
--
GitLab
From a9856152a2ed9352853122d91b738634708efd12 Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 12:47:57 +0100
Subject: [PATCH 7/8] broker: add force deactivate all method to forcefully
change the database flag of active brokers
---
CHANGES.md | 1 +
weatherdb/broker.py | 11 ++++++++++-
weatherdb/cli.py | 10 ++++++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index 4f62f73..b6a7472 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,6 +6,7 @@
- cli: add version parameter
- multiprocessing: make process count configurable through user configuration
- broker: prevent broker inactivation by other broker
+- broker: add force deactivate all method to forcefully change the database flag of active brokers
## Version 1.2.0
diff --git a/weatherdb/broker.py b/weatherdb/broker.py
index a0f55c1..0d7bc72 100644
--- a/weatherdb/broker.py
+++ b/weatherdb/broker.py
@@ -665,4 +665,13 @@ class Broker(object):
yield self
finally:
self._deactivate()
- atexit.unregister(self._deactivate)
\ No newline at end of file
+ atexit.unregister(self._deactivate)
+
+ def force_deactivate_all(self):
+ """Forcefully set the active broker flag in the database to deactivated.
+
+ This is useful if the broker got exited before it could deactivate itself.
+ """
+ if self.is_active:
+ self._deactivate()
+ self.set_setting("broker_active", "False")
\ No newline at end of file
diff --git a/weatherdb/cli.py b/weatherdb/cli.py
index 7c8979b..2a7a625 100644
--- a/weatherdb/cli.py
+++ b/weatherdb/cli.py
@@ -222,6 +222,16 @@ def set_db_version():
else:
click.echo("aborting setting db version")
+@cli.command(short_help="Forcefully set the active broker flag in the database to deactivated. This is useful if the broker got exited before it could deactivate itself. (!!!Only use this if you're sure that the database did all the necessary updates!!!)")
+def force_deactivate_all_broker():
+ click.echo(dedent("""
+ Are you sure that there is no more broker running? This could lead to problems as multiple brokers could run at the same time contradicting themself."""))
+ if click.confirm("Are you sure you want to set the activation flag in the database to deactivated?"):
+ click.echo("deactivating the brokers flag")
+ broker = weatherdb.broker.Broker()
+ broker.force_deactivate_all()
+ else:
+ click.echo("aborting the deactivation of the brokers flag")
# cli
# ---------------------------------------
--
GitLab
From 78c39f293c8b90d874d7164945561d1c6bd7877a Mon Sep 17 00:00:00 2001
From: maxschmi <maxschm@yahoo.de>
Date: Wed, 15 Jan 2025 13:19:06 +0100
Subject: [PATCH 8/8] config: change LINEAR_INTERPOLATION_LIMIT to be a
category
---
CHANGES.md | 1 +
weatherdb/config/config_default.ini | 7 ++++---
weatherdb/station/StationBases.py | 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index b6a7472..6d53d13 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -3,6 +3,7 @@
## Version 1.2.1
- fix: filllup linear regression did not take configuration value and therefor did never work
+- config: change LINEAR_INTERPOLATION_LIMIT to be a category
- cli: add version parameter
- multiprocessing: make process count configurable through user configuration
- broker: prevent broker inactivation by other broker
diff --git a/weatherdb/config/config_default.ini b/weatherdb/config/config_default.ini
index f0d9131..362f4a0 100644
--- a/weatherdb/config/config_default.ini
+++ b/weatherdb/config/config_default.ini
@@ -143,6 +143,7 @@ RASTER_BUFFER_CRS = ${weatherdb:HORIZON_CRS}
; The default is 1999-01-01
MIN_DATE = 1999-01-01
+[weatherdb:LINEAR_INTERPOLATION_LIMIT]
; When there are still NAs after filling with neighboring stations, the module can use linear interpolation to fill the gaps
; The limit defines the maximum interval that is allowed to fill with linear interpolation
; The value is given with a unit, e.g. 3 days, 30 minutes, 1 hour
@@ -150,11 +151,11 @@ MIN_DATE = 1999-01-01
; for precipitation, where the timeseries resolution is 10 minutes, 30 minutes means 3 missing values are interpolated, but not more.
; If there shouldn't be any linear interpolation, set the limit to 0
; For precipitation, the default is 1 hour
-LINEAR_INTERPOLATION_LIMIT_P = 1 hour
+P = 1 hour
; For temperature, the default is 2 days
-LINEAR_INTERPOLATION_LIMIT_T = 2 days
+T = 2 days
; For Evapotranspiration, the default is 2 days
-LINEAR_INTERPOLATION_LIMIT_ET = 2 days
+ET = 2 days
[weatherdb:max_fillup_distance]
diff --git a/weatherdb/station/StationBases.py b/weatherdb/station/StationBases.py
index 30dcb1b..2cb3cf5 100644
--- a/weatherdb/station/StationBases.py
+++ b/weatherdb/station/StationBases.py
@@ -1493,8 +1493,8 @@ class StationBase:
# linear interpolation for the last missing values
lr_limit = config.get(
- "weatherdb",
- f"LINEAR_INTERPOLATION_LIMIT_{self._para_base}",
+ "weatherdb:linear_interpolation_limit",
+ self._para_base,
fallback="0")
if lr_limit != "0":
sql_format_dict.update(dict(
--
GitLab