From 554ec1c0f0444d649f7e94c4ebe67e61d74a0f8f Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 27 May 2024 22:07:09 +0900 Subject: [PATCH] nixos/tests: add postgresql wal2json test This test should ensure wal2json functions. I'm planning to upgrade wal2json, so it seems nice to have a test here. It passes on my machine. --- nixos/tests/all-tests.nix | 1 + nixos/tests/postgresql-wal2json.nix | 60 ++++++++++++++++ nixos/tests/postgresql/wal2json/LICENSE | 27 +++++++ nixos/tests/postgresql/wal2json/README.md | 11 +++ nixos/tests/postgresql/wal2json/example2.out | 74 ++++++++++++++++++++ nixos/tests/postgresql/wal2json/example2.sql | 31 ++++++++ nixos/tests/postgresql/wal2json/example3.out | 12 ++++ nixos/tests/postgresql/wal2json/example3.sql | 26 +++++++ pkgs/servers/sql/postgresql/ext/wal2json.nix | 7 +- 9 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 nixos/tests/postgresql-wal2json.nix create mode 100644 nixos/tests/postgresql/wal2json/LICENSE create mode 100644 nixos/tests/postgresql/wal2json/README.md create mode 100644 nixos/tests/postgresql/wal2json/example2.out create mode 100644 nixos/tests/postgresql/wal2json/example2.sql create mode 100644 nixos/tests/postgresql/wal2json/example3.out create mode 100644 nixos/tests/postgresql/wal2json/example3.sql diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 6c9ff9fb9c20..92fa7fbbfd7e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -805,6 +805,7 @@ in { postgresql-jit = handleTest ./postgresql-jit.nix {}; postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; postgresql-tls-client-cert = handleTest ./postgresql-tls-client-cert.nix {}; + postgresql-wal2json = handleTest ./postgresql-wal2json.nix {}; powerdns = handleTest ./powerdns.nix {}; powerdns-admin = handleTest ./powerdns-admin.nix {}; power-profiles-daemon = handleTest ./power-profiles-daemon.nix {}; diff --git a/nixos/tests/postgresql-wal2json.nix b/nixos/tests/postgresql-wal2json.nix new file mode 100644 index 000000000000..043ad48cbc6e --- /dev/null +++ b/nixos/tests/postgresql-wal2json.nix @@ -0,0 +1,60 @@ +{ + system ? builtins.currentSystem, + config ? { }, + pkgs ? import ../.. { inherit system config; }, + postgresql ? null, +}: + +let + makeTest = import ./make-test-python.nix; + # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. + makeTestAttribute = name: { + inherit name; + value = makePostgresqlWal2jsonTest pkgs."${name}"; + }; + + makePostgresqlWal2jsonTest = + postgresqlPackage: + makeTest { + name = "postgresql-wal2json-${postgresqlPackage.name}"; + meta.maintainers = with pkgs.lib.maintainers; [ euank ]; + + nodes.machine = { + services.postgresql = { + package = postgresqlPackage; + enable = true; + extraPlugins = with postgresqlPackage.pkgs; [ wal2json ]; + settings = { + wal_level = "logical"; + max_replication_slots = "10"; + max_wal_senders = "10"; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("postgresql") + machine.succeed( + "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example2.sql} postgres > /tmp/example2.out" + ) + machine.succeed( + "diff ${./postgresql/wal2json/example2.out} /tmp/example2.out" + ) + machine.succeed( + "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example3.sql} postgres > /tmp/example3.out" + ) + machine.succeed( + "diff ${./postgresql/wal2json/example3.out} /tmp/example3.out" + ) + ''; + }; + +in +# By default, create one test per postgresql version +if postgresql == null then + builtins.listToAttrs ( + map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs)) + ) +# but if postgresql is set, we're being made as a passthru test for a specific postgres + wal2json version, just run one +else + makePostgresqlWal2jsonTest postgresql diff --git a/nixos/tests/postgresql/wal2json/LICENSE b/nixos/tests/postgresql/wal2json/LICENSE new file mode 100644 index 000000000000..e3e82163fc09 --- /dev/null +++ b/nixos/tests/postgresql/wal2json/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013-2024, Euler Taveira de Oliveira +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the Euler Taveira de Oliveira nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nixos/tests/postgresql/wal2json/README.md b/nixos/tests/postgresql/wal2json/README.md new file mode 100644 index 000000000000..796bf810d757 --- /dev/null +++ b/nixos/tests/postgresql/wal2json/README.md @@ -0,0 +1,11 @@ +Data in this folder taken from the wal2json README's examples [here](https://github.com/eulerto/wal2json/tree/75629c2e1e81a12350cc9d63782fc53252185d8d#sql-functions) + +They are used under the terms of the BSD-3 License, a copy of which is included +in this directory. + +These files have been lightly modified in order to make their output more reproducible. + +Changes: +- `\o /dev/null` has been added before commands that print LSNs since LSNs aren't reproducible +- `now()` has been replaced with a hardcoded timestamp string for reproducibility +- The test is run with `--quiet`, and the expected output has been trimmed accordingly diff --git a/nixos/tests/postgresql/wal2json/example2.out b/nixos/tests/postgresql/wal2json/example2.out new file mode 100644 index 000000000000..0a089e112270 --- /dev/null +++ b/nixos/tests/postgresql/wal2json/example2.out @@ -0,0 +1,74 @@ +init +{ + "change": [ + { + "kind": "message", + "transactional": false, + "prefix": "wal2json", + "content": "this non-transactional message will be delivered even if you rollback the transaction" + } + ] +} +{ + "change": [ + { + "kind": "insert", + "schema": "public", + "table": "table2_with_pk", + "columnnames": ["a", "b", "c"], + "columntypes": ["integer", "character varying(30)", "timestamp without time zone"], + "columnvalues": [1, "Backup and Restore", "2018-03-27 12:05:29.914496"] + } + ,{ + "kind": "insert", + "schema": "public", + "table": "table2_with_pk", + "columnnames": ["a", "b", "c"], + "columntypes": ["integer", "character varying(30)", "timestamp without time zone"], + "columnvalues": [2, "Tuning", "2018-03-27 12:05:29.914496"] + } + ,{ + "kind": "insert", + "schema": "public", + "table": "table2_with_pk", + "columnnames": ["a", "b", "c"], + "columntypes": ["integer", "character varying(30)", "timestamp without time zone"], + "columnvalues": [3, "Replication", "2018-03-27 12:05:29.914496"] + } + ,{ + "kind": "message", + "transactional": true, + "prefix": "wal2json", + "content": "this message will be delivered" + } + ,{ + "kind": "delete", + "schema": "public", + "table": "table2_with_pk", + "oldkeys": { + "keynames": ["a", "c"], + "keytypes": ["integer", "timestamp without time zone"], + "keyvalues": [1, "2018-03-27 12:05:29.914496"] + } + } + ,{ + "kind": "delete", + "schema": "public", + "table": "table2_with_pk", + "oldkeys": { + "keynames": ["a", "c"], + "keytypes": ["integer", "timestamp without time zone"], + "keyvalues": [2, "2018-03-27 12:05:29.914496"] + } + } + ,{ + "kind": "insert", + "schema": "public", + "table": "table2_without_pk", + "columnnames": ["a", "b", "c"], + "columntypes": ["integer", "numeric(5,2)", "text"], + "columnvalues": [1, 2.34, "Tapir"] + } + ] +} +stop diff --git a/nixos/tests/postgresql/wal2json/example2.sql b/nixos/tests/postgresql/wal2json/example2.sql new file mode 100644 index 000000000000..ec474381bb4d --- /dev/null +++ b/nixos/tests/postgresql/wal2json/example2.sql @@ -0,0 +1,31 @@ +CREATE TABLE table2_with_pk (a SERIAL, b VARCHAR(30), c TIMESTAMP NOT NULL, PRIMARY KEY(a, c)); +CREATE TABLE table2_without_pk (a SERIAL, b NUMERIC(5,2), c TEXT); + +SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'wal2json'); + +BEGIN; +INSERT INTO table2_with_pk (b, c) VALUES('Backup and Restore', '2018-03-27 12:05:29.914496'); +INSERT INTO table2_with_pk (b, c) VALUES('Tuning', '2018-03-27 12:05:29.914496'); +INSERT INTO table2_with_pk (b, c) VALUES('Replication', '2018-03-27 12:05:29.914496'); + +-- Avoid printing wal LSNs since they're not reproducible, so harder to assert on +\o /dev/null +SELECT pg_logical_emit_message(true, 'wal2json', 'this message will be delivered'); +SELECT pg_logical_emit_message(true, 'pgoutput', 'this message will be filtered'); +\o + +DELETE FROM table2_with_pk WHERE a < 3; +\o /dev/null +SELECT pg_logical_emit_message(false, 'wal2json', 'this non-transactional message will be delivered even if you rollback the transaction'); +\o + +INSERT INTO table2_without_pk (b, c) VALUES(2.34, 'Tapir'); +-- it is not added to stream because there isn't a pk or a replica identity +UPDATE table2_without_pk SET c = 'Anta' WHERE c = 'Tapir'; +COMMIT; + +SELECT data FROM pg_logical_slot_get_changes('test_slot', NULL, NULL, 'pretty-print', '1', 'add-msg-prefixes', 'wal2json'); +SELECT 'stop' FROM pg_drop_replication_slot('test_slot'); + +DROP TABLE table2_with_pk; +DROP TABLE table2_without_pk; diff --git a/nixos/tests/postgresql/wal2json/example3.out b/nixos/tests/postgresql/wal2json/example3.out new file mode 100644 index 000000000000..e20d2a8aefd7 --- /dev/null +++ b/nixos/tests/postgresql/wal2json/example3.out @@ -0,0 +1,12 @@ +init +{"action":"M","transactional":false,"prefix":"wal2json","content":"this non-transactional message will be delivered even if you rollback the transaction"} +{"action":"B"} +{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":1},{"name":"b","type":"character varying(30)","value":"Backup and Restore"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]} +{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":2},{"name":"b","type":"character varying(30)","value":"Tuning"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]} +{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":3},{"name":"b","type":"character varying(30)","value":"Replication"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]} +{"action":"M","transactional":true,"prefix":"wal2json","content":"this message will be delivered"} +{"action":"D","schema":"public","table":"table3_with_pk","identity":[{"name":"a","type":"integer","value":1},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]} +{"action":"D","schema":"public","table":"table3_with_pk","identity":[{"name":"a","type":"integer","value":2},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]} +{"action":"I","schema":"public","table":"table3_without_pk","columns":[{"name":"a","type":"integer","value":1},{"name":"b","type":"numeric(5,2)","value":2.34},{"name":"c","type":"text","value":"Tapir"}]} +{"action":"C"} +stop diff --git a/nixos/tests/postgresql/wal2json/example3.sql b/nixos/tests/postgresql/wal2json/example3.sql new file mode 100644 index 000000000000..6d94e261f51a --- /dev/null +++ b/nixos/tests/postgresql/wal2json/example3.sql @@ -0,0 +1,26 @@ +CREATE TABLE table3_with_pk (a SERIAL, b VARCHAR(30), c TIMESTAMP NOT NULL, PRIMARY KEY(a, c)); +CREATE TABLE table3_without_pk (a SERIAL, b NUMERIC(5,2), c TEXT); + +SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'wal2json'); + +BEGIN; +INSERT INTO table3_with_pk (b, c) VALUES('Backup and Restore', '2019-12-29 04:58:34.806671'); +INSERT INTO table3_with_pk (b, c) VALUES('Tuning', '2019-12-29 04:58:34.806671'); +INSERT INTO table3_with_pk (b, c) VALUES('Replication', '2019-12-29 04:58:34.806671'); +\o /dev/null +SELECT pg_logical_emit_message(true, 'wal2json', 'this message will be delivered'); +SELECT pg_logical_emit_message(true, 'pgoutput', 'this message will be filtered'); +DELETE FROM table3_with_pk WHERE a < 3; +SELECT pg_logical_emit_message(false, 'wal2json', 'this non-transactional message will be delivered even if you rollback the transaction'); +\o + +INSERT INTO table3_without_pk (b, c) VALUES(2.34, 'Tapir'); +-- it is not added to stream because there isn't a pk or a replica identity +UPDATE table3_without_pk SET c = 'Anta' WHERE c = 'Tapir'; +COMMIT; + +SELECT data FROM pg_logical_slot_get_changes('test_slot', NULL, NULL, 'format-version', '2', 'add-msg-prefixes', 'wal2json'); +SELECT 'stop' FROM pg_drop_replication_slot('test_slot'); + +DROP TABLE table3_with_pk; +DROP TABLE table3_without_pk; diff --git a/pkgs/servers/sql/postgresql/ext/wal2json.nix b/pkgs/servers/sql/postgresql/ext/wal2json.nix index 26d4cb0c1541..32483849019c 100644 --- a/pkgs/servers/sql/postgresql/ext/wal2json.nix +++ b/pkgs/servers/sql/postgresql/ext/wal2json.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ lib, callPackage, stdenv, fetchFromGitHub, postgresql }: stdenv.mkDerivation rec { pname = "wal2json"; @@ -20,6 +20,11 @@ stdenv.mkDerivation rec { install -D -t $out/share/postgresql/extension sql/*.sql ''; + passthru.tests.wal2json = lib.recurseIntoAttrs (callPackage ../../../../../nixos/tests/postgresql-wal2json.nix { + inherit (stdenv) system; + inherit postgresql; + }); + meta = with lib; { description = "PostgreSQL JSON output plugin for changeset extraction"; homepage = "https://github.com/eulerto/wal2json";