# HG changeset patch
# User Christophe de Vienne <christophe@cdevienne.info>
# Date 1671463790 -3600
# Mon Dec 19 16:29:50 2022 +0100
# Node ID 7f80dcd7d60780a5613ea6fd3c9fa3627eeeca46
# Parent 12b3a76b616a1231c43294bdc7f6d1f251e14c57
3.3.0 release notes and docs
diff --git a/VERSIONS.json b/VERSIONS.json
--- a/VERSIONS.json
+++ b/VERSIONS.json
@@ -1,5 +1,5 @@
{
- "xbus-api": "default",
- "xbus": "3.3.0-rc.2",
- "go-xbus": "v3.3.0"
+ "xbus-api": "3.3",
+ "xbus": "3.3.0",
+ "go-xbus": "v3.3.1"
}
diff --git a/build-cmd-doc/go.sum b/build-cmd-doc/go.sum
--- a/build-cmd-doc/go.sum
+++ b/build-cmd-doc/go.sum
@@ -889,7 +889,10 @@
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
xbus.io/go-xbus/v3 v3.0.0-20210322163351-ce43a2e8be95/go.mod h1:7peOPZ7Z9iXm6ilJOH8BohQz7imB96eGMS1oz5cunuk=
+xbus.io/go-xbus/v3 v3.0.0-20221216155809-144c12808280/go.mod h1:7peOPZ7Z9iXm6ilJOH8BohQz7imB96eGMS1oz5cunuk=
xbus.io/go-xbus/v3 v3.2.0 h1:0bP6P2/vdr/501VNnlj1Ugr3wndxqSeAn30BqsoCgY4=
xbus.io/go-xbus/v3 v3.2.0/go.mod h1:aFbqMkUwdhk12aepYcDsVlgxXyZG0oHCAXuOe8U9l9E=
xbus.io/go-xbus/v3 v3.3.0 h1:8BXWZBqL7iSmxlVly9FkmF1jWlzeCItiDmICFku4ATw=
xbus.io/go-xbus/v3 v3.3.0/go.mod h1:7peOPZ7Z9iXm6ilJOH8BohQz7imB96eGMS1oz5cunuk=
+xbus.io/go-xbus/v3 v3.3.1 h1:kAKxTyhu8643IppHivYUUrEgPLYCqirIGvqk/0uh+68=
+xbus.io/go-xbus/v3 v3.3.1/go.mod h1:7peOPZ7Z9iXm6ilJOH8BohQz7imB96eGMS1oz5cunuk=
diff --git a/source/HISTORY.rst b/source/HISTORY.rst
--- a/source/HISTORY.rst
+++ b/source/HISTORY.rst
@@ -2,9 +2,181 @@
History
=======
+3.3.0 (2022-12-16)
+------------------
+
+3.3.0-rc.6 (2022-09-09)
+-----------------------
+
+- sentinel: Add an optional periodic 'onlinestatus' survey.
+
+3.3.0-rc.5 (2022-07-18)
+-----------------------
+
+- envelope-storage-ng: disable fragments merging, which fixes some case of
+ failure when fetching mid-size envelopes.
+
+3.3.0-rc.4 (2022-06-27)
+-----------------------
+
+- service director fix: ProcessingEnd was not called in some case, leading
+ to RUNNING process with no jobs (which should never happen).
+
+- service director: log warnings in the process on timeout errors
+
+- envelope-storage-ng: fix handling an initial 'null' fragment
+
+3.3.0-rc.3 (2022-06-21)
+-----------------------
+
+- xbusctl pipeline:
+
+ - when serializing a pipeline, sort nodes and edges and remove empty values.
+ It makes ``set-all`` more stable.
+
+ - ``set-all``: save new versions even if not different from the previous version.
+
+- service 'director'
+
+ - fix job runner: multiple jobs could be saved as 'running' for a single actor
+
+ - add SERVERTOOBUSY error messages to the process logs, as warnings.
+
+- implement the new api 'Debug'. Can be used through the ``xbusctl debug info``
+ command.
+
+- http webhook: fix error handling when posting separate messages
+
+- log storage: sort logs by time
+
+- service 'metrics'
+
+ - fix log storage size gauge
+
+3.3.0-rc.2 (2022-05-12)
+-----------------------
+
+- pipelines: New 'sourcematch' properties:
+
+ - ``byactor`` (boolean, default ``false``): if true, event types are ignored
+
+ - ``priority`` (string): force the priority to 'low' or 'high'
+
+- http.webhook:
+
+ - add a 'payload-type' setting. See the documentation for more details.
+ - add a 'headers' setting.
+
+- merge 3.2.10 changes
+
+3.3.0-rc.1 (2022-02-01)
+-----------------------
+
+- Merge v3.2.3 to v3.2.9 changes
+- Update the go toolchain to 1.16
+- Update go-xbus, gogo-nrpc & nats.
+- Code cleaning: enable more golangci linters and clean the code accordingly
+
+- Fix EnvelopeStorageNG when saving multiple fragmented envelopes (:issue:`191`)
+- ps export: add the 'created_at' process property (:issue:`189`)
+
+3.3.0-alpha.1 (2021-03-15)
+--------------------------
+
+- Refactoring of the 'director' service for better code readability.
+
+- Storage:
+
+ - Add a metrics gathering EnvelopeStorage layer. It is automatically enabled
+ when xbusd metrics are 'on'.
+
+ - Add a new EnvelopeStorage implementation with better performances:
+ EnvelopeStorageNG. Can be enabled with ``--storage.envelope.impl "sql-ng"``.
+
+ - Add a fallback mechanism for EnvelopeStorage, to make envelope storage
+ transition smoothless. Can be enabled with ``--storage.envelope.fallback``.
+
+ - Add a cached EnvelopeStorage layer that works on top of a persistent
+ implementation. Can be enabled with ``--storage.envelope.cache.enabled``.
+ The cache size is 1Gio max, which can be changed with
+ ``--storage.envelope.cache.max-size``.
+
3.2
===
+3.2.10 (2022-02-22)
+-------------------
+
+- metrics: fix a heavy load on the db after a 'last' collection. Occured only if
+ the metrics where not, or no longer, collected.
+
+3.2.9 (2022-01-27)
+------------------
+
+- metrics: better handling of slow metrics (process count & storage stats)
+
+ In some cases (mainly big databases) the slowness of metric fetching
+ led to a complete loss of those metrics.
+
+3.2.8 (2021-12-06)
+------------------
+
+- process query: fix a 'nats: maximum payload exceeded' occuring in some rare
+ cases.
+
+3.2.7 (2021-06-29)
+------------------
+
+- pipeline save: fix saving edges updates.
+
+3.2.6 (2021-06-03)
+------------------
+
+- metrics: precalculate process summary and storage stats to avoid timeout on
+ metrics. If no new data is available, the last result is collected again up
+ to 30 seconds after its initial collection (see
+ :ref:`administration-metrics`).
+
+3.2.5 (2021-04-27)
+------------------
+
+- Update go-xbus to v3.2.1, fixing a routine leak in xbus-http (and xbus-client
+ and xbusctl).
+
+3.2.4 (2021-04-02)
+------------------
+
+- xbusd: Fix replaying a process that initially failed with 'no matching
+ pipeline'.
+
+3.2.3 (2021-03-15)
+------------------
+
+- xbusd: Generate certificates without "Common Name" which is nowaday
+ deprecated and not supported in go 1.16.
+
+3.2.2 (2021-03-09)
+------------------
+
+- xbusctl actor list: fix 'STATUS' that was always 'online'
+- xbusd: remove noisy useless logs
+- xbusd: fix replayed process that remaining in "RUNNING" state
+
+3.2.1 (2021-03-04)
+------------------
+
+- graph verification: forbid 'sourcematch' on consumer and worker nodes (:issue:`76`)
+
+- xbusd:
+
+ - add '--log-level' and '--quiet' settings (:issue:`71`).
+ - account accept: fix a crash when no CSR is pending (:issue:`154`)
+ - Fix a crash in control.Process.ExportStream if the process pipeline no longer
+ exists (:issue:`186`).
+ - process storage: optimize transaction isolation of process summary. It should
+ help high loads not to be impacted by the metrics gathering.
+
+
3.2.0 (2020-12-16)
------------------
diff --git a/source/administration/envelope-storage.rst b/source/administration/envelope-storage.rst
new file mode 100644
--- /dev/null
+++ b/source/administration/envelope-storage.rst
@@ -0,0 +1,86 @@
+.. _administration-envelope-storage:
+
+Envelope Storage
+================
+
+Starting with version 3.3.0, Xbus provides a more efficient way to store the
+envelopes in the database. This new storage is not enabled by default.
+
+A typical post-3.3.0 migration configuration would be:
+
+.. code-block:: yaml
+
+ storage:
+ envelope:
+ impl: sql-ng
+ fallback: sql
+ cache:
+ enabled: true
+ max-size: 1G
+
+Primary Storage
+---------------
+
+Setting the primary storage, ie the storage for new envelopes, is done with the
+option ``storage.envelope.impl``. It can take one of the following values:
+
+- ``sql`` (default). The historical envelope storage. It stores the envelope
+ content in different tables, allowing an access to the envelope inner data.
+
+ The tables used are:
+
+ - ``envelope`` : store the envelope metadata
+ - ``event``: store the message metadata (id, sum...)
+ - ``item``: store data chunks of the messages
+
+ This storage has some performances issues, as it has to split then rebuild
+ the fragments all the time.
+
+- ``sql-ng``. This storage stores directly the serialized envelope fragments,
+ allowing a much faster read and write.
+
+ It uses the following tables:
+
+ - ``envelope_storage`` stores the envelope metadata
+ - ``envelope_storage_data`` stores the fragments
+
+
+Fallback storage
+----------------
+
+As there is no way to migrate envelopes from a storage to another, a mechanism
+of fallback storage is provided.
+
+It allows to specify a secondary storage, which will be used for reading
+envelopes that are not found in the primary storage.
+
+The setting is ``storage.envelope.fallback``, and the possible values are the
+same as ``storage.envelope.impl``.
+
+
+Cache
+-----
+
+A cache can be enabled on top of the chosen storage. Two options are available:
+
+``storage.envelope.cache.enabled``
+ Enable the cache
+
+``storage.envelope.cache.max-size``
+ (default "1G") Set the cache size
+
+Metrics
+-------
+
+When xbusd metrics are enabled, new metrics are available:
+
+- ``xbus_storage_calls``, a summary with the following dynamic labels:
+
+ - ``name``: The storage name ("cache", "default"...)
+
+ - ``implementation``: The storage implementation ("cache", "sql", "sql-ng")
+
+ - ``function``: The function. Possible values are:
+
+ - ``ReadEnvelope``: Read a fragment
+ - ``StoreEnvelope``: Store a fragment
diff --git a/source/administration/index.rst b/source/administration/index.rst
--- a/source/administration/index.rst
+++ b/source/administration/index.rst
@@ -7,4 +7,5 @@
pipeline
cert-maintenance
db-maintenance
+ envelope-storage
metrics
diff --git a/source/go-xbus.rst b/source/go-xbus.rst
--- a/source/go-xbus.rst
+++ b/source/go-xbus.rst
@@ -20,6 +20,21 @@
Versions
========
+3.3.1 (2022-12-16)
+------------------
+
+- api: regen the control api (with added Debug service)
+
+3.3.0 (2022-02-01)
+------------------
+
+- Update gogo-nrpc, switch to nats.go (instead of go-nats)
+
+3.2.1 (2021-01-06)
+------------------
+
+- Fix a routine leak in envelope emission
+
3.2.0 (2020-12-16)
------------------
diff --git a/source/migration.rst b/source/migration.rst
--- a/source/migration.rst
+++ b/source/migration.rst
@@ -21,6 +21,9 @@
the following environment variable: ``GODEBUG=x509ignoreCN=0``.
+Certificates
+''''''''''''
+
The binaries of this version are compiled with go-1.16. In go 1.16, the x509
certificates 'CommonName' attribute is deprecated. Instead the SANs extension
should be used.
@@ -37,7 +40,13 @@
GODEBUG=x509ignoreCN=0
-.. _migration-to-3.3.0:
+Envelope Storage
+''''''''''''''''
+
+A new storage for envelopes is available, and switching to it is recommended.
+Learn more: :ref:`administration-envelope-storage`.
+
+.. _migration-to-3.2.0:
To 3.2.3
~~~~~~~~
diff --git a/source/releasenotes/3.3.0.rst b/source/releasenotes/3.3.0.rst
new file mode 100644
--- /dev/null
+++ b/source/releasenotes/3.3.0.rst
@@ -0,0 +1,111 @@
+3.3.0 (2022-07-18)
+==================
+
+After a long series of release candidates, Xbus 3.3.0 is finally available.
+
+Changes
+-------
+
+The focus of this version is on performance & stability, with a few new features.
+
+New features
+~~~~~~~~~~~~
+
+- The new envelope storage implementations provide better performance for
+ storing and reading envelopes of all sizes. Learn more in the
+ :ref:`administration-envelope-storage` section.
+
+- ``ps export`` now exports the process creation date/time.
+
+- Pipelines: New 'sourcematch' properties:
+
+ - ``byactor`` (boolean, default ``false``): if true, event types are ignored
+
+ - ``priority`` (string): force the priority to 'low' or 'high'
+
+- New API "Debug": provides useful debug informations. Run ``xbusctl debug info``
+ to discover what is available.
+
+- Service sentinel: Add an optional periodic 'onlinestatus' survey.
+
+ This survey will make sure no actor stays offline when the client is actually
+ running, which can happen when a client has replicas. To enable it, use the
+ flag ``--sentinel-survey-onlinestatus-delay``, or the config.ini setting
+ ``sentinel.survey-onlinestatus-delay``.
+
+Improvements
+~~~~~~~~~~~~
+
+- The 'director' service is completely refactored for better code readability,
+ resulting in fewer bugs, and much more reliable behavior.
+
+- ``pipeline`` exports now sort nodes and remove empty values, making
+ ``set-all`` much more stable.
+
+- Many dependencies are updated to more recent versions, including nats &
+ gogo-nrpc.
+
+- The Go toolchain updated to go 1.16. Because of the long release candidate
+ phase, it is already a little outdated, and we will address this soon.
+
+
+Documentation
+-------------
+
+Documentation is online:
+
+ https://docs.xbus.io/
+
+Downloads
+---------
+
+Source and binaries are available on our download area:
+
+- https://dl.xbus.io/xbus/src/xbus-3.3.0-src.tar.xz
+
+version "personal":
+
+- https://dl.xbus.io/xbus/personal/xbus-3.3.0-linux_amd64.tar.xz
+- https://dl.xbus.io/xbus/personal/xbus-3.3.0-windows_amd64.tar.xz
+- https://dl.xbus.io/xbus/personal/xbus-3.3.0-darwin_amd64.tar.xz
+
+version "enterprise":
+
+- https://dl.xbus.io/xbus/enterprise/xbus-3.3.0-enterprise-linux_amd64.tar.xz
+- https://dl.xbus.io/xbus/enterprise/xbus-3.3.0-enterprise-windows_amd64.tar.xz
+- https://dl.xbus.io/xbus/enterprise/xbus-3.3.0-enterprise-darwin_amd64.tar.xz
+
+version "lts":
+
+- https://dl.xbus.io/xbus/lts/xbus-3.3.0-lts-linux_amd64.tar.xz
+- https://dl.xbus.io/xbus/lts/xbus-3.3.0-lts-windows_amd64.tar.xz
+- https://dl.xbus.io/xbus/lts/xbus-3.3.0-lts-darwin_amd64.tar.xz
+
+Docker images
+-------------
+
+Get the docker images:
+
+version "personal":
+
+ - orus/xbusd:3.3.0
+ - orus/xbusctl:3.3.0
+ - orus/xbus-client:3.3.0
+ - orus/xbus-http:3.3.0
+ - orus/xbus-fullenv:3.3.0
+
+version "enterprise":
+
+ - quay.orus.io/xbus-enterprise/xbusd:3.3.0-enterprise
+ - quay.orus.io/xbus-enterprise/xbusctl:3.3.0-enterprise
+ - quay.orus.io/xbus-enterprise/xbus-client:3.3.0-enterprise
+ - quay.orus.io/xbus-enterprise/xbus-http:3.3.0-enterprise
+ - quay.orus.io/xbus-enterprise/xbus-fullenv:3.3.0-enterprise
+
+version "LTS":
+
+ - quay.orus.io/xbus-lts/xbusd:3.3.0-lts
+ - quay.orus.io/xbus-lts/xbusctl:3.3.0-lts
+ - quay.orus.io/xbus-lts/xbus-client:3.3.0-lts
+ - quay.orus.io/xbus-lts/xbus-http:3.3.0-lts
+ - quay.orus.io/xbus-lts/xbus-fullenv:3.3.0-lts
diff --git a/source/releasenotes/index.rst b/source/releasenotes/index.rst
--- a/source/releasenotes/index.rst
+++ b/source/releasenotes/index.rst
@@ -4,6 +4,7 @@
=============
.. toctree::
+ 3.3.0
3.3.0-rc.6
3.3.0-rc.5
3.3.0-rc.4