diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.4.1 + +- Include migrations in the package + # 0.4.0 - SMP server implementation and [SMP protocol](https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md) changes: diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -13,11 +13,9 @@ ## SimpleXMQ roadmap -- Streams - high performance message queues. See [Streams RFC](https://github.com/simplex-chat/simplexmq/blob/master/rfcs/2021-02-28-streams.md) for details. -- "Small" connection groups, when each message will be sent by the SMP agent to multiple connections with a single client command. See [Groups RFC](https://github.com/simplex-chat/simplexmq/blob/master/rfcs/2021-03-18-groups.md) for details. -- SMP agents cluster to share connections and message management by multiple agents (for example, it would enable multi-device use for [simplex-chat](https://github.com/simplex-chat/simplex-chat)). - SMP queue redundancy and rotation in SMP agent duplex connections. -- "Large" groups design and implementation. +- SMP agents synchronization to share connections and messages between multiple agents (it would allow using multiple devices for [simplex-chat](https://github.com/simplex-chat/simplex-chat)). +- Streams - high performance message queues. See [Streams RFC](https://github.com/simplex-chat/simplexmq/blob/master/rfcs/2021-02-28-streams.md) for details. ## Components @@ -50,13 +48,14 @@ ## Using SMP server and SMP agent -You can either run your own SMP server locally or deploy using [Linode StackScript](#deploy-smp-server-on-linode), or try local SMP agent with the deployed demo server: +You can either run your own SMP server locally or deploy using [Linode StackScript](#deploy-smp-server-on-linode), or try local SMP agent with the deployed servers: -`smp1.simplex.im:5223#pLdiGvm0jD1CMblnov6Edd/391OrYsShw+RgdfR0ChA=` +`smp2.simplex.im#z5W2QLQ1Br3Yd6CoWg7bIq1bHdwK7Y8bEiEXBs/WfAg=` (London, UK) +`smp3.simplex.im#nxc7HnrnM8dOKgkMp008ub/9o9LXJlxlMrMpR+mfMQw=` (Fremont, CA) It's the easiest to try SMP agent via a prototype [simplex-chat](https://github.com/simplex-chat/simplex-chat) terminal UI. -[linode](https://cloud.linode.com/stackscripts/748014) +[Linode](https://cloud.linode.com/stackscripts/748014) ## Deploy SMP server on Linode @@ -78,7 +77,7 @@ Please submit an [issue](https://github.com/simplex-chat/simplexmq/issues) if any problems occur. -[linode](https://marketplace.digitalocean.com/apps/simplex-server) +[DigitalOcean](https://marketplace.digitalocean.com/apps/simplex-server) ## Deploy SMP server on DigitalOcean diff --git a/migrations/20210101_initial.sql b/migrations/20210101_initial.sql new file mode 100644 --- /dev/null +++ b/migrations/20210101_initial.sql @@ -0,0 +1,117 @@ +CREATE TABLE IF NOT EXISTS servers( + host TEXT NOT NULL, + port TEXT NOT NULL, + key_hash BLOB, + PRIMARY KEY (host, port) +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS rcv_queues( + host TEXT NOT NULL, + port TEXT NOT NULL, + rcv_id BLOB NOT NULL, + conn_alias BLOB NOT NULL, + rcv_private_key BLOB NOT NULL, + snd_id BLOB, + snd_key BLOB, + decrypt_key BLOB NOT NULL, + verify_key BLOB, + status TEXT NOT NULL, + PRIMARY KEY (host, port, rcv_id), + FOREIGN KEY (host, port) REFERENCES servers (host, port), + FOREIGN KEY (conn_alias) + REFERENCES connections (conn_alias) + ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + UNIQUE (host, port, snd_id) +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS snd_queues( + host TEXT NOT NULL, + port TEXT NOT NULL, + snd_id BLOB NOT NULL, + conn_alias BLOB NOT NULL, + snd_private_key BLOB NOT NULL, + encrypt_key BLOB NOT NULL, + sign_key BLOB NOT NULL, + status TEXT NOT NULL, + PRIMARY KEY (host, port, snd_id), + FOREIGN KEY (host, port) REFERENCES servers (host, port), + FOREIGN KEY (conn_alias) + REFERENCES connections (conn_alias) + ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS connections( + conn_alias BLOB NOT NULL, + rcv_host TEXT, + rcv_port TEXT, + rcv_id BLOB, + snd_host TEXT, + snd_port TEXT, + snd_id BLOB, + last_internal_msg_id INTEGER NOT NULL, + last_internal_rcv_msg_id INTEGER NOT NULL, + last_internal_snd_msg_id INTEGER NOT NULL, + last_external_snd_msg_id INTEGER NOT NULL, + last_rcv_msg_hash BLOB NOT NULL, + last_snd_msg_hash BLOB NOT NULL, + PRIMARY KEY (conn_alias), + FOREIGN KEY (rcv_host, rcv_port, rcv_id) REFERENCES rcv_queues (host, port, rcv_id), + FOREIGN KEY (snd_host, snd_port, snd_id) REFERENCES snd_queues (host, port, snd_id) +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS messages( + conn_alias BLOB NOT NULL, + internal_id INTEGER NOT NULL, + internal_ts TEXT NOT NULL, + internal_rcv_id INTEGER, + internal_snd_id INTEGER, + body TEXT NOT NULL, -- deprecated + PRIMARY KEY (conn_alias, internal_id), + FOREIGN KEY (conn_alias) + REFERENCES connections (conn_alias) + ON DELETE CASCADE, + FOREIGN KEY (conn_alias, internal_rcv_id) + REFERENCES rcv_messages (conn_alias, internal_rcv_id) + ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + FOREIGN KEY (conn_alias, internal_snd_id) + REFERENCES snd_messages (conn_alias, internal_snd_id) + ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS rcv_messages( + conn_alias BLOB NOT NULL, + internal_rcv_id INTEGER NOT NULL, + internal_id INTEGER NOT NULL, + external_snd_id INTEGER NOT NULL, + external_snd_ts TEXT NOT NULL, + broker_id BLOB NOT NULL, + broker_ts TEXT NOT NULL, + rcv_status TEXT NOT NULL, + ack_brocker_ts TEXT, + ack_sender_ts TEXT, + internal_hash BLOB NOT NULL, + external_prev_snd_hash BLOB NOT NULL, + integrity BLOB NOT NULL, + PRIMARY KEY (conn_alias, internal_rcv_id), + FOREIGN KEY (conn_alias, internal_id) + REFERENCES messages (conn_alias, internal_id) + ON DELETE CASCADE +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS snd_messages( + conn_alias BLOB NOT NULL, + internal_snd_id INTEGER NOT NULL, + internal_id INTEGER NOT NULL, + snd_status TEXT NOT NULL, + sent_ts TEXT, + delivered_ts TEXT, + internal_hash BLOB NOT NULL, + PRIMARY KEY (conn_alias, internal_snd_id), + FOREIGN KEY (conn_alias, internal_id) + REFERENCES messages (conn_alias, internal_id) + ON DELETE CASCADE +) WITHOUT ROWID; diff --git a/migrations/20210624_confirmations.sql b/migrations/20210624_confirmations.sql new file mode 100644 --- /dev/null +++ b/migrations/20210624_confirmations.sql @@ -0,0 +1,9 @@ +CREATE TABLE conn_confirmations ( + confirmation_id BLOB NOT NULL PRIMARY KEY, + conn_alias BLOB NOT NULL REFERENCES connections ON DELETE CASCADE, + sender_key BLOB NOT NULL, + sender_conn_info BLOB NOT NULL, + accepted INTEGER NOT NULL, + own_conn_info BLOB, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +) WITHOUT ROWID; diff --git a/migrations/20210809_snd_messages.sql b/migrations/20210809_snd_messages.sql new file mode 100644 --- /dev/null +++ b/migrations/20210809_snd_messages.sql @@ -0,0 +1,3 @@ +ALTER TABLE messages ADD msg_body BLOB NOT NULL DEFAULT x''; -- this field replaces body TEXT +-- TODO possibly migrate the data from body if it is possible in migration +ALTER TABLE snd_messages ADD previous_msg_hash BLOB NOT NULL DEFAULT x''; diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,9 @@ +# SQLite database migrations + +These migrations are [embedded](../src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs) into the executable and run when SMP agent starts (as a separate executable or as a part of [simplex-chat](https://github.com/simplex-chat/simplex-chat) app). + +Migration file names must have a format `YYYYMMDD-name.sql` - they will be executed in the order or lexicographic sorting of the names, the files with any other extension than `.sql` are ignored. + +The proposed approach is to minimize the number of migrations and merge them together when possible, to align with the agent releases. + +**Please note**: Adding or editing migrations will NOT update the migrations embedded into the executable, unless the [Migrations](../src/Simplex/Messaging/Agent/Store/SQLite/Migrations.hs) module is rebuilt - use `stack build --force-dirty` (in addition to edited files it seems to rebuild the files with TH splices and their dependencies, not all files as with `stack clean`). diff --git a/simplexmq.cabal b/simplexmq.cabal --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack -- --- hash: e25d53dc1b12c8bcdf029091182613198be5271348ea19174b6532d820fe1fc9 +-- hash: 706d2f9155c3f3be0f08ea0d6c8954c0e2b9a6e22615f7b19499a3a349af7cc9 name: simplexmq -version: 0.4.0 +version: 0.4.1 synopsis: SimpleXMQ message broker description: This package includes <./docs/Simplex-Messaging-Server.html server>, <./docs/Simplex-Messaging-Client.html client> and @@ -28,6 +28,10 @@ extra-source-files: README.md CHANGELOG.md + migrations/20210101_initial.sql + migrations/20210624_confirmations.sql + migrations/20210809_snd_messages.sql + migrations/README.md library exposed-modules: diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs --- a/src/Simplex/Messaging/Agent.hs +++ b/src/Simplex/Messaging/Agent.hs @@ -106,7 +106,7 @@ where smpAgent :: forall c m'. (Transport c, MonadUnliftIO m', MonadReader Env m') => TProxy c -> m' () smpAgent _ = runTransportServer started tcpPort $ \(h :: c) -> do - liftIO $ putLn h "Welcome to SMP v0.4.0 agent" + liftIO $ putLn h "Welcome to SMP v0.4.1 agent" c <- getAgentClient logConnection c True race_ (connectClient h c) (runAgentClient c) diff --git a/src/Simplex/Messaging/Transport.hs b/src/Simplex/Messaging/Transport.hs --- a/src/Simplex/Messaging/Transport.hs +++ b/src/Simplex/Messaging/Transport.hs @@ -221,7 +221,7 @@ major (SMPVersion a b _ _) = (a, b) currentSMPVersion :: SMPVersion -currentSMPVersion = "0.4.0.0" +currentSMPVersion = "0.4.1.0" serializeSMPVersion :: SMPVersion -> ByteString serializeSMPVersion (SMPVersion a b c d) = B.intercalate "." [bshow a, bshow b, bshow c, bshow d] diff --git a/tests/SMPAgentClient.hs b/tests/SMPAgentClient.hs --- a/tests/SMPAgentClient.hs +++ b/tests/SMPAgentClient.hs @@ -189,7 +189,7 @@ testSMPAgentClientOn port' client = do runTransportClient agentTestHost port' $ \h -> do line <- liftIO $ getLn h - if line == "Welcome to SMP v0.4.0 agent" + if line == "Welcome to SMP v0.4.1 agent" then client h else error $ "wrong welcome message: " <> B.unpack line