From 9557ff11f5c7bcffceec4621a5077f9457d4552f Mon Sep 17 00:00:00 2001 From: Brandur Date: Sun, 8 Nov 2026 11:41:55 -0600 Subject: [PATCH 1/2] Add MySQL support to River Add MySQL drivers, migrations, client integration, and CI coverage across Go, Java, JavaScript, Ruby, and Rust. Preserve job uniqueness, transaction, notification, and cancellation behavior across the supported databases. Include bulk-operation optimizations and reusable cross-database benchmarks. --- .github/workflows/ci.yaml | 24 + .github/workflows/conformance.yaml | 38 +- .github/workflows/java.yaml | 34 + .github/workflows/js.yaml | 33 + .github/workflows/ruby.yaml | 53 +- .github/workflows/rust.yaml | 35 +- Makefile | 78 +- client.go | 26 +- client_test.go | 43 + cmd/river/benchcompare/database.go | 168 ++ cmd/river/benchcompare/main.go | 195 ++ cmd/river/benchcompare/measure.go | 125 + cmd/river/benchcompare/measure_test.go | 46 + cmd/river/go.mod | 7 + cmd/river/go.sum | 28 +- cmd/river/rivercli/command.go | 59 +- cmd/river/rivercli/command_test.go | 43 + cmd/river/rivercli/driver_procurer.go | 22 + cmd/river/rivercli/river_cli.go | 2 +- cmd/river/rivercli/river_cli_test.go | 2 +- conformance/go.sum | 20 +- docs/development.md | 171 +- go.sum | 16 +- go.work | 1 + internal/cmd/syncrustmigrations/main.go | 4 + internal/dblist/db_list.go | 16 +- internal/maintenance/job_rescuer_test.go | 51 +- .../sqlite_notification_cleaner.go | 9 +- java/DIFFERENCES.md | 2 +- java/README.md | 30 +- java/bin/check-packages/main.go | 5 +- java/bin/check-packages/main_test.go | 1 + java/bin/github-release.py | 2 +- java/bin/sync-migrations/main.go | 3 +- java/bin/sync-migrations/main_test.go | 4 +- java/cli/pom.xml | 1 + .../java/com/riverqueue/cli/MigrationCli.java | 26 +- java/conformance/pom.xml | 1 + java/docs/development.md | 14 +- java/pom.xml | 1 + java/river/pom.xml | 1 + .../src/main/java/com/riverqueue/Client.java | 26 +- .../main/java/com/riverqueue/Database.java | 21 +- .../main/java/com/riverqueue/JobQuery.java | 48 +- .../src/main/java/com/riverqueue/Jobs.java | 26 +- .../main/java/com/riverqueue/Migrator.java | 33 +- .../src/main/java/com/riverqueue/MySql.java | 149 ++ .../src/main/java/com/riverqueue/Queues.java | 5 +- .../main/java/com/riverqueue/RunHandle.java | 54 +- .../src/main/java/com/riverqueue/Sql.java | 60 + .../main/java/com/riverqueue/WorkContext.java | 5 +- .../mysql/001_create_river_migration.down.sql | 1 + .../mysql/001_create_river_migration.up.sql | 8 + .../mysql/002_initial_schema.down.sql | 2 + .../migration/mysql/002_initial_schema.up.sql | 42 + .../003_river_job_tags_non_null.down.sql | 1 + .../mysql/003_river_job_tags_non_null.up.sql | 2 + .../mysql/004_pending_and_more.down.sql | 21 + .../mysql/004_pending_and_more.up.sql | 48 + .../005_migration_unique_client.down.sql | 35 + .../mysql/005_migration_unique_client.up.sql | 60 + .../migration/mysql/006_bulk_unique.down.sql | 9 + .../migration/mysql/006_bulk_unique.up.sql | 45 + ...tbox_sqlite_jsonb_and_sql_cleanup.down.sql | 57 + ...outbox_sqlite_jsonb_and_sql_cleanup.up.sql | 44 + .../mysql/008_job_id_autoincrement.down.sql | 2 + .../mysql/008_job_id_autoincrement.up.sql | 2 + .../main/resources/com/riverqueue/mysql.sql | 298 +++ .../test/java/com/riverqueue/ClientTest.java | 111 +- .../test/java/com/riverqueue/MySqlTest.java | 102 + .../com/riverqueue/PeerCancellationTest.java | 10 +- .../java/com/riverqueue/PeerDecodingTest.java | 5 +- .../com/riverqueue/ProtocolStorageTest.java | 12 +- .../java/com/riverqueue/RunHandleTest.java | 5 +- .../java/com/riverqueue/SchedulingTest.java | 4 +- .../java/com/riverqueue/TestDatabase.java | 11 +- job_list_params.go | 30 +- job_list_params_test.go | 26 +- js/CHANGELOG.md | 4 + js/README.md | 9 +- js/cli/package.json | 1 + js/cli/src/database.ts | 5 +- js/cli/src/migrate-commands.ts | 40 +- js/cli/src/run.test.ts | 2 +- js/docs/development.md | 8 + js/driver/mysql/README.md | 56 + js/driver/mysql/etc/driver-mysql.api.md | 33 + js/driver/mysql/package.json | 60 + js/driver/mysql/src/codecs.ts | 131 + .../mysql/src/driver.integration.test.ts | 994 ++++++++ js/driver/mysql/src/driver.ts | 1339 ++++++++++ js/driver/mysql/src/index.ts | 11 + js/driver/mysql/tsconfig.json | 10 + js/etc/riverqueue.api.md | 5 + js/etc/riverqueue.unstable-driver.api.md | 5 + js/migrate/etc/migrate.api.md | 37 +- js/migrate/migrations/manifest.json | 67 + .../main/001_create_river_migration.down.sql | 1 + .../main/001_create_river_migration.up.sql | 8 + .../mysql/main/002_initial_schema.down.sql | 2 + .../mysql/main/002_initial_schema.up.sql | 42 + .../main/003_river_job_tags_non_null.down.sql | 1 + .../main/003_river_job_tags_non_null.up.sql | 2 + .../mysql/main/004_pending_and_more.down.sql | 21 + .../mysql/main/004_pending_and_more.up.sql | 48 + .../main/005_migration_unique_client.down.sql | 35 + .../main/005_migration_unique_client.up.sql | 60 + .../mysql/main/006_bulk_unique.down.sql | 9 + .../mysql/main/006_bulk_unique.up.sql | 45 + ...tbox_sqlite_jsonb_and_sql_cleanup.down.sql | 57 + ...outbox_sqlite_jsonb_and_sql_cleanup.up.sql | 44 + .../main/008_job_id_autoincrement.down.sql | 2 + .../main/008_job_id_autoincrement.up.sql | 2 + js/migrate/src/bundle.ts | 4 +- js/migrate/src/index.ts | 3 + js/migrate/src/migrator.ts | 31 +- js/migrate/src/mysql.ts | 199 ++ js/package.json | 1 + js/pnpm-lock.yaml | 51 +- js/scripts/api-reports.mjs | 5 + js/scripts/check-licenses.mjs | 1 + js/scripts/check-packages.mjs | 24 +- js/scripts/check-readme-snippets.mjs | 4 + js/scripts/sync-migrations.mjs | 1 + js/src/pilot.ts | 5 + js/tsconfig.tests.json | 3 + js/typedoc.json | 1 + riverdbtest/riverdbtest.go | 3 +- riverdriver/go.sum | 4 +- riverdriver/river_driver_interface.go | 16 +- riverdriver/riverdatabasesql/go.sum | 20 +- .../internal/dbsqlc/river_job.sql.go | 4 +- .../river_database_sql_driver.go | 9 +- .../riverdrivertest/client_cancel_test.go | 3 + .../riverdrivertest/client_job_delete_test.go | 2 +- .../riverdrivertest/client_job_update_test.go | 103 + .../riverdrivertest/driver_client_test.go | 128 +- riverdriver/riverdrivertest/driver_test.go | 61 + riverdriver/riverdrivertest/executor_tx.go | 77 +- riverdriver/riverdrivertest/go.mod | 7 + riverdriver/riverdrivertest/go.sum | 32 +- riverdriver/riverdrivertest/job_delete.go | 66 +- riverdriver/riverdrivertest/job_insert.go | 167 +- riverdriver/riverdrivertest/job_read.go | 66 +- riverdriver/riverdrivertest/job_update.go | 143 +- riverdriver/riverdrivertest/leader.go | 23 + riverdriver/riverdrivertest/listener.go | 8 +- riverdriver/riverdrivertest/migration.go | 39 +- riverdriver/riverdrivertest/notification.go | 2 +- riverdriver/riverdrivertest/queue.go | 17 + .../riverdrivertest/riverdrivertest.go | 50 +- riverdriver/riverdrivertest/sql_fragments.go | 28 + riverdriver/rivermysql/example_mysql_test.go | 125 + riverdriver/rivermysql/go.mod | 31 + riverdriver/rivermysql/go.sum | 54 + .../internal/dbbatch/job_complete.sql | 10 + .../internal/dbbatch/job_insert.sql | 29 + .../internal/dbbatch/job_insert_upsert.sql | 20 + .../internal/dbbatch/job_rescue.sql | 26 + .../internal/dbbatch/job_set_state.sql | 38 + .../internal/dbbatch/job_unique_key.sql | 16 + .../internal/dbbatch/notification_insert.sql | 8 + .../internal/dbfragment/json_contains.sql | 5 + riverdriver/rivermysql/internal/dbsqlc/db.go | 24 + .../rivermysql/internal/dbsqlc/models.go | 82 + .../rivermysql/internal/dbsqlc/river_job.sql | 487 ++++ .../internal/dbsqlc/river_job.sql.go | 1656 ++++++++++++ .../internal/dbsqlc/river_leader.sql | 60 + .../internal/dbsqlc/river_leader.sql.go | 178 ++ .../internal/dbsqlc/river_migration.sql | 80 + .../internal/dbsqlc/river_migration.sql.go | 375 +++ .../internal/dbsqlc/river_notification.sql | 52 + .../internal/dbsqlc/river_notification.sql.go | 148 ++ .../internal/dbsqlc/river_queue.sql | 101 + .../internal/dbsqlc/river_queue.sql.go | 323 +++ .../rivermysql/internal/dbsqlc/schema.sql | 79 + .../rivermysql/internal/dbsqlc/schema.sql.go | 226 ++ .../rivermysql/internal/dbsqlc/sqlc.yaml | 43 + .../rivermysql/internal/dbtype/json.go | 37 + .../rivermysql/internal/dbtype/json_test.go | 55 + .../main/001_create_river_migration.down.sql | 1 + .../main/001_create_river_migration.up.sql | 8 + .../main/002_initial_schema.down.sql | 2 + .../migration/main/002_initial_schema.up.sql | 42 + .../main/003_river_job_tags_non_null.down.sql | 1 + .../main/003_river_job_tags_non_null.up.sql | 2 + .../main/004_pending_and_more.down.sql | 21 + .../main/004_pending_and_more.up.sql | 48 + .../main/005_migration_unique_client.down.sql | 35 + .../main/005_migration_unique_client.up.sql | 60 + .../migration/main/006_bulk_unique.down.sql | 9 + .../migration/main/006_bulk_unique.up.sql | 45 + ...tbox_sqlite_jsonb_and_sql_cleanup.down.sql | 57 + ...outbox_sqlite_jsonb_and_sql_cleanup.up.sql | 44 + .../main/008_job_id_autoincrement.down.sql | 2 + .../main/008_job_id_autoincrement.up.sql | 2 + riverdriver/rivermysql/river_mysql_driver.go | 2240 +++++++++++++++++ .../rivermysql/river_mysql_driver_batch.go | 299 +++ .../river_mysql_driver_batch_test.go | 425 ++++ .../river_mysql_driver_benchmark_test.go | 236 ++ .../rivermysql/river_mysql_driver_json.go | 157 ++ .../rivermysql/river_mysql_driver_test.go | 1123 +++++++++ .../rivermysql/river_mysql_listener.go | 316 +++ riverdriver/riverpgxv5/go.sum | 16 +- .../riverpgxv5/internal/dbsqlc/river_job.sql | 4 +- .../internal/dbsqlc/river_job.sql.go | 6 +- riverdriver/riverpgxv5/river_pgx_v5_driver.go | 15 +- riverdriver/riversqlite/go.sum | 20 +- .../riversqlite/river_sqlite_driver.go | 9 +- rivermigrate/river_migrate.go | 10 +- rivershared/go.sum | 16 +- .../riversharedtest/riversharedtest.go | 41 + rivershared/sqlctemplate/sqlc_template.go | 195 +- .../sqlctemplate/sqlc_template_test.go | 250 +- ruby/CHANGELOG.md | 4 + ruby/Gemfile | 1 + ruby/Gemfile.lock | 3 + ruby/Makefile | 2 +- ruby/README.md | 21 +- ruby/docs/conformance.md | 23 +- ruby/docs/development.md | 6 + ruby/docs/migrations.md | 15 +- ruby/driver/riverqueue-activerecord/Gemfile | 1 + .../riverqueue-activerecord/Gemfile.lock | 3 + .../riverqueue-activerecord/lib/driver.rb | 24 +- .../spec/driver_spec.rb | 35 + ruby/driver/riverqueue-sequel/Gemfile | 1 + ruby/driver/riverqueue-sequel/Gemfile.lock | 3 + ruby/driver/riverqueue-sequel/lib/driver.rb | 21 +- .../riverqueue-sequel/spec/driver_spec.rb | 17 + ruby/lib/driver.rb | 1 + ruby/lib/driver/mysql_runtime.rb | 413 +++ ruby/lib/driver/notification_listener.rb | 25 +- ruby/lib/driver/runtime.rb | 16 +- ruby/lib/migration_cli.rb | 10 +- ruby/lib/migrator.rb | 60 +- ruby/migration/manifest.json | 16 + .../main/001_create_river_migration.down.sql | 1 + .../main/001_create_river_migration.up.sql | 8 + .../mysql/main/002_initial_schema.down.sql | 2 + .../mysql/main/002_initial_schema.up.sql | 42 + .../main/003_river_job_tags_non_null.down.sql | 1 + .../main/003_river_job_tags_non_null.up.sql | 2 + .../mysql/main/004_pending_and_more.down.sql | 21 + .../mysql/main/004_pending_and_more.up.sql | 48 + .../main/005_migration_unique_client.down.sql | 35 + .../main/005_migration_unique_client.up.sql | 60 + .../mysql/main/006_bulk_unique.down.sql | 9 + .../mysql/main/006_bulk_unique.up.sql | 45 + ...tbox_sqlite_jsonb_and_sql_cleanup.down.sql | 57 + ...outbox_sqlite_jsonb_and_sql_cleanup.up.sql | 44 + .../main/008_job_id_autoincrement.down.sql | 2 + .../main/008_job_id_autoincrement.up.sql | 2 + ruby/scripts/check_coverage.rb | 6 +- ruby/scripts/sync_migrations.rb | 2 +- ruby/sig/driver.rbs | 12 + ruby/sig/migrator.rbs | 2 + ruby/sig/mysql_runtime.rbs | 17 + ruby/spec/conformance_spec.rb | 35 +- ruby/spec/driver_runtime_shared_examples.rb | 6 +- ruby/spec/driver_shared_examples.rb | 31 +- ruby/spec/migration_export_spec.rb | 8 +- ruby/spec/migrator_shared_examples.rb | 24 +- ruby/spec/migrator_spec.rb | 116 +- ruby/spec/mysql_spec.rb | 177 ++ .../notification_receiver_shared_examples.rb | 43 +- ruby/spec/notification_receiver_spec.rb | 2 +- ruby/spec/notification_runtime_spec.rb | 23 + ruby/spec/row_decoding_shared_examples.rb | 12 + ruby/spec/spec_helper.rb | 10 +- ruby/spec/support/client_test_database.rb | 39 + .../migrations/mysql/test/001_test.down.sql | 1 + .../migrations/mysql/test/001_test.up.sql | 1 + ruby/spec/support/river_test_schema.rb | 2 + ruby/spec/support/test_database.rb | 7 +- rust/CHANGELOG.md | 4 + rust/Cargo.lock | 1 + rust/Cargo.toml | 3 +- rust/README.md | 40 +- rust/docs/development.md | 22 + rust/riverqueue-cli/Cargo.toml | 3 +- rust/riverqueue-cli/README.md | 6 +- rust/riverqueue-cli/src/migrate.rs | 28 +- rust/riverqueue-cli/src/migrate_get.rs | 17 +- rust/riverqueue-cli/tests/migrate_get.rs | 22 +- rust/riverqueue-migrate/Cargo.toml | 1 + rust/riverqueue-migrate/README.md | 15 +- .../main/001_create_river_migration.down.sql | 1 + .../main/001_create_river_migration.up.sql | 8 + .../mysql/main/002_initial_schema.down.sql | 2 + .../mysql/main/002_initial_schema.up.sql | 42 + .../main/003_river_job_tags_non_null.down.sql | 1 + .../main/003_river_job_tags_non_null.up.sql | 2 + .../mysql/main/004_pending_and_more.down.sql | 21 + .../mysql/main/004_pending_and_more.up.sql | 48 + .../main/005_migration_unique_client.down.sql | 35 + .../main/005_migration_unique_client.up.sql | 60 + .../mysql/main/006_bulk_unique.down.sql | 9 + .../mysql/main/006_bulk_unique.up.sql | 45 + ...tbox_sqlite_jsonb_and_sql_cleanup.down.sql | 57 + ...outbox_sqlite_jsonb_and_sql_cleanup.up.sql | 44 + .../main/008_job_id_autoincrement.down.sql | 2 + .../main/008_job_id_autoincrement.up.sql | 2 + rust/riverqueue-migrate/src/lib.rs | 15 +- rust/riverqueue-migrate/src/mysql.rs | 264 ++ rust/riverqueue-test/Cargo.toml | 1 + rust/riverqueue/Cargo.toml | 4 +- rust/riverqueue/README.md | 39 +- rust/riverqueue/src/__private.rs | 96 + rust/riverqueue/src/client/completer.rs | 79 +- rust/riverqueue/src/client/insert.rs | 187 +- rust/riverqueue/src/client/jobs.rs | 13 +- rust/riverqueue/src/client/mod.rs | 12 +- rust/riverqueue/src/client/notifier.rs | 99 + rust/riverqueue/src/client/producer.rs | 36 + rust/riverqueue/src/client/queues.rs | 12 +- rust/riverqueue/src/client/run.rs | 14 +- rust/riverqueue/src/client/tests.rs | 4 +- rust/riverqueue/src/database.rs | 159 ++ rust/riverqueue/src/database/erased.rs | 41 +- rust/riverqueue/src/database/mysql.rs | 1566 ++++++++++++ .../src/database/mysql/connection.rs | 114 + rust/riverqueue/src/database/mysql/tests.rs | 928 +++++++ .../src/database/mysql/tests/conformance.rs | 354 +++ rust/riverqueue/src/lib.rs | 8 +- rust/riverqueue/src/maintenance/cleaner.rs | 104 +- rust/riverqueue/src/maintenance/elector.rs | 45 + rust/riverqueue/src/maintenance/maintainer.rs | 17 + rust/riverqueue/src/maintenance/mod.rs | 34 +- rust/riverqueue/src/maintenance/rescuer.rs | 97 + rust/riverqueue/src/maintenance/scheduler.rs | 79 +- rust/riverqueue/src/pilot.rs | 13 + rust/riverqueue/src/storage.rs | 59 +- rust/riverqueue/src/storage/mysql.rs | 277 ++ rust/riverqueue/tests/database_contract.rs | 23 + 335 files changed, 23976 insertions(+), 626 deletions(-) create mode 100644 cmd/river/benchcompare/database.go create mode 100644 cmd/river/benchcompare/main.go create mode 100644 cmd/river/benchcompare/measure.go create mode 100644 cmd/river/benchcompare/measure_test.go create mode 100644 cmd/river/rivercli/command_test.go create mode 100644 java/river/src/main/java/com/riverqueue/MySql.java create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql create mode 100644 java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql create mode 100644 java/river/src/main/resources/com/riverqueue/mysql.sql create mode 100644 java/river/src/test/java/com/riverqueue/MySqlTest.java create mode 100644 js/driver/mysql/README.md create mode 100644 js/driver/mysql/etc/driver-mysql.api.md create mode 100644 js/driver/mysql/package.json create mode 100644 js/driver/mysql/src/codecs.ts create mode 100644 js/driver/mysql/src/driver.integration.test.ts create mode 100644 js/driver/mysql/src/driver.ts create mode 100644 js/driver/mysql/src/index.ts create mode 100644 js/driver/mysql/tsconfig.json create mode 100644 js/migrate/migrations/mysql/main/001_create_river_migration.down.sql create mode 100644 js/migrate/migrations/mysql/main/001_create_river_migration.up.sql create mode 100644 js/migrate/migrations/mysql/main/002_initial_schema.down.sql create mode 100644 js/migrate/migrations/mysql/main/002_initial_schema.up.sql create mode 100644 js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql create mode 100644 js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql create mode 100644 js/migrate/migrations/mysql/main/004_pending_and_more.down.sql create mode 100644 js/migrate/migrations/mysql/main/004_pending_and_more.up.sql create mode 100644 js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql create mode 100644 js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql create mode 100644 js/migrate/migrations/mysql/main/006_bulk_unique.down.sql create mode 100644 js/migrate/migrations/mysql/main/006_bulk_unique.up.sql create mode 100644 js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql create mode 100644 js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql create mode 100644 js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql create mode 100644 js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql create mode 100644 js/migrate/src/mysql.ts create mode 100644 riverdriver/rivermysql/example_mysql_test.go create mode 100644 riverdriver/rivermysql/go.mod create mode 100644 riverdriver/rivermysql/go.sum create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_complete.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_insert.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_rescue.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_set_state.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql create mode 100644 riverdriver/rivermysql/internal/dbbatch/notification_insert.sql create mode 100644 riverdriver/rivermysql/internal/dbfragment/json_contains.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/db.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/models.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_job.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_leader.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_migration.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_notification.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_queue.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/schema.sql create mode 100644 riverdriver/rivermysql/internal/dbsqlc/schema.sql.go create mode 100644 riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml create mode 100644 riverdriver/rivermysql/internal/dbtype/json.go create mode 100644 riverdriver/rivermysql/internal/dbtype/json_test.go create mode 100644 riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql create mode 100644 riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql create mode 100644 riverdriver/rivermysql/migration/main/002_initial_schema.down.sql create mode 100644 riverdriver/rivermysql/migration/main/002_initial_schema.up.sql create mode 100644 riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql create mode 100644 riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql create mode 100644 riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql create mode 100644 riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql create mode 100644 riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql create mode 100644 riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql create mode 100644 riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql create mode 100644 riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql create mode 100644 riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql create mode 100644 riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql create mode 100644 riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql create mode 100644 riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql create mode 100644 riverdriver/rivermysql/river_mysql_driver.go create mode 100644 riverdriver/rivermysql/river_mysql_driver_batch.go create mode 100644 riverdriver/rivermysql/river_mysql_driver_batch_test.go create mode 100644 riverdriver/rivermysql/river_mysql_driver_benchmark_test.go create mode 100644 riverdriver/rivermysql/river_mysql_driver_json.go create mode 100644 riverdriver/rivermysql/river_mysql_driver_test.go create mode 100644 riverdriver/rivermysql/river_mysql_listener.go create mode 100644 ruby/lib/driver/mysql_runtime.rb create mode 100644 ruby/migration/mysql/main/001_create_river_migration.down.sql create mode 100644 ruby/migration/mysql/main/001_create_river_migration.up.sql create mode 100644 ruby/migration/mysql/main/002_initial_schema.down.sql create mode 100644 ruby/migration/mysql/main/002_initial_schema.up.sql create mode 100644 ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql create mode 100644 ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql create mode 100644 ruby/migration/mysql/main/004_pending_and_more.down.sql create mode 100644 ruby/migration/mysql/main/004_pending_and_more.up.sql create mode 100644 ruby/migration/mysql/main/005_migration_unique_client.down.sql create mode 100644 ruby/migration/mysql/main/005_migration_unique_client.up.sql create mode 100644 ruby/migration/mysql/main/006_bulk_unique.down.sql create mode 100644 ruby/migration/mysql/main/006_bulk_unique.up.sql create mode 100644 ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql create mode 100644 ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql create mode 100644 ruby/migration/mysql/main/008_job_id_autoincrement.down.sql create mode 100644 ruby/migration/mysql/main/008_job_id_autoincrement.up.sql create mode 100644 ruby/sig/mysql_runtime.rbs create mode 100644 ruby/spec/mysql_spec.rb create mode 100644 ruby/spec/support/migrations/mysql/test/001_test.down.sql create mode 100644 ruby/spec/support/migrations/mysql/test/001_test.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/001_create_river_migration.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/001_create_river_migration.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql create mode 100644 rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql create mode 100644 rust/riverqueue-migrate/src/mysql.rs create mode 100644 rust/riverqueue/src/database/mysql.rs create mode 100644 rust/riverqueue/src/database/mysql/connection.rs create mode 100644 rust/riverqueue/src/database/mysql/tests.rs create mode 100644 rust/riverqueue/src/database/mysql/tests/conformance.rs create mode 100644 rust/riverqueue/src/storage/mysql.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b57f15a21..379687111 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,6 +98,7 @@ jobs: # latest Postgres version: - go-version: "1.26" postgres-version: 18 + fail-fast: false timeout-minutes: 5 @@ -149,6 +150,29 @@ jobs: if: ${{ !matrix.primary }} run: make test TEST_DATABASE=postgres + build_and_test_mysql: + env: + GOTOOLCHAIN: local + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version: "1.27" + + # MySQL is pre-installed on GitHub-hosted Ubuntu runners. + - name: Start MySQL + run: | + sudo systemctl start mysql + mysql -u root -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED BY ''; FLUSH PRIVILEGES;" + + - name: Test with race detector + run: make test/race TEST_DATABASE=mysql + build_and_test_sqlite: name: build_and_test_sqlite (${{ matrix.go-version }}${{ matrix.primary && ', race' || '' }}) env: diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml index a674d975b..51d347774 100644 --- a/.github/workflows/conformance.yaml +++ b/.github/workflows/conformance.yaml @@ -44,9 +44,27 @@ env: jobs: port_fixtures: - name: Port fixture tests + name: Port fixture tests (SQLite and MySQL) runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 + + env: + RIVER_RUST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + RIVER_TEST_DATABASE_URL: mysql://root@127.0.0.1:3306/ + TEST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 steps: - uses: actions/checkout@v7 @@ -77,16 +95,32 @@ jobs: - name: Rust fixture tests run: make test/rust/conformance + - name: Rust MySQL fixture tests + run: make test/rust/conformance/mysql + - name: JavaScript fixture tests run: make test/js/conformance + - name: JavaScript MySQL fixture tests + run: make test/js/conformance/mysql + - name: Java fixture tests run: make test/java/conformance + - name: Java MySQL fixture tests + run: make test/java/conformance/mysql + - uses: ruby/setup-ruby@v1 with: working-directory: ruby bundler-cache: true - name: Ruby fixture tests + env: + TEST_DATABASE: sqlite run: make test/ruby/conformance + + - name: Ruby MySQL fixture tests + env: + TEST_MYSQL_URL: mysql2://root@127.0.0.1:3306/ + run: make test/ruby/conformance/mysql diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 64c92b1ef..f723082bf 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -12,6 +12,7 @@ on: - "Makefile" - "conformance/**" - "java/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" pull_request: @@ -22,6 +23,7 @@ on: - "Makefile" - "conformance/**" - "java/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" workflow_dispatch: @@ -132,3 +134,35 @@ jobs: - name: Postgres client, worker, and migration tests run: make test/java/postgres + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_TEST_DATABASE_URL: mysql://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ./.github/actions/setup-java + - uses: actions/setup-go@v6 + with: + go-version-file: go.work + - run: make test/java/mysql diff --git a/.github/workflows/js.yaml b/.github/workflows/js.yaml index a8fd9620f..a4cf2a69d 100644 --- a/.github/workflows/js.yaml +++ b/.github/workflows/js.yaml @@ -263,3 +263,36 @@ jobs: - run: node cli/dist/bin.js migrate-up --database-url "$TEST_DATABASE_URL" - run: pnpm run test:integration + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + TEST_MYSQL_URL: mysql://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ./.github/actions/setup-js + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + - run: make test/js/mysql + working-directory: . diff --git a/.github/workflows/ruby.yaml b/.github/workflows/ruby.yaml index 226221a65..5462a600a 100644 --- a/.github/workflows/ruby.yaml +++ b/.github/workflows/ruby.yaml @@ -35,8 +35,8 @@ defaults: jobs: checks: - # Check coverage from the two backends without adding another CI job. - needs: [test, test_sqlite] + # Check coverage from all backends without adding another CI job. + needs: [test, test_sqlite, mysql] runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -209,3 +209,52 @@ jobs: env: RAILS_VERSION: "~> 8.1.0" run: bundle install && bundle exec rspec + + mysql: + name: Test (MySQL ${{ matrix.mysql-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_REQUIRE_DATABASES: "1" + TEST_DATABASE: mysql + TEST_MYSQL_URL: mysql2://root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "4.0" + working-directory: ruby + bundler-cache: true + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + # Escaping a maximum-size output can double its 32 MiB JSON payload. + - name: Allow maximum-size job output + run: docker exec "${{ job.services.mysql.id }}" mysql -uroot -e 'SET GLOBAL max_allowed_packet = 134217728' + - run: make generate/fixtures spec/core + - uses: actions/upload-artifact@v7 + if: matrix.mysql-version == '8.4' + with: + name: ruby-coverage-mysql + path: ruby/coverage/.resultset.json + include-hidden-files: true + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 466107542..0f48275ca 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -12,6 +12,7 @@ on: - "Makefile" - "conformance/**" - "internal/cmd/syncrustmigrations/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" - "rust/**" @@ -22,6 +23,7 @@ on: - "Makefile" - "conformance/**" - "internal/cmd/syncrustmigrations/**" + - "riverdriver/rivermysql/migration/**" - "riverdriver/riverpgxv5/migration/**" - "riverdriver/riversqlite/migration/**" - "rust/**" @@ -70,7 +72,7 @@ jobs: with: tool: cargo-deny@0.20.2 - - name: Lint, including Postgres-only and SQLite-only builds + - name: Lint, including each backend on its own run: make lint/rust - name: Documentation and examples @@ -208,3 +210,34 @@ jobs: run: PGPASSWORD=postgres createdb -h localhost -U postgres river_rust_test - name: Rust unit, SQLite, and Postgres tests run: make test/rust + + mysql: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + mysql-version: ["8.0", "8.4"] + env: + RIVER_RUST_MYSQL_URL: mysql://root:root@127.0.0.1:3306/ + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_ROOT_HOST: "%" + ports: ["3306:3306"] + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -proot" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy,rustfmt + - uses: actions/setup-go@v7 + with: + go-version-file: go.work + - run: make test/rust/mysql diff --git a/Makefile b/Makefile index f31a5091e..a18e2758d 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ generate/rust-migrations: ## Sync database migrations to Rust .PHONY: generate/sqlc generate/sqlc: ## Generate sqlc cd riverdriver/riverdatabasesql/internal/dbsqlc && $(SQLC) generate + cd riverdriver/rivermysql/internal/dbsqlc && $(SQLC) generate cd riverdriver/riverpgxv5/internal/dbsqlc && $(SQLC) generate cd riverdriver/riversqlite/internal/dbsqlc && $(SQLC) generate @@ -80,19 +81,25 @@ TEST_DATABASE ?= all # Only filter the shared driver suite. Other packages have SQLite-named tests # that use Postgres or mocks and should stay in the regular test run. +mysql_test_pattern := ^(Test.*MySQL|Example_mysql) sqlite_test_pattern := ^(Test.*(LibSQL|SQLite|Turso)|Example_(libSQL|sqlite|turso)) test_submodules := $(submodules) legacy_driver_test_flags := -run '/WithTx$$' -ifeq ($(TEST_DATABASE),postgres) - test_submodules := $(filter-out %/riverdriver/riversqlite,$(submodules)) - driver_test_flags := -skip '$(sqlite_test_pattern)' - legacy_driver_test_flags += -skip '$(sqlite_test_pattern)' +ifeq ($(TEST_DATABASE),mysql) + export RIVER_MYSQL_TESTS_ENABLED := 1 + test_submodules := $(filter %/riverdriver/riverdrivertest %/riverdriver/rivermysql,$(submodules)) + driver_test_flags := -run '$(mysql_test_pattern)' + legacy_driver_test_flags := -run '$(mysql_test_pattern)/WithTx$$' +else ifeq ($(TEST_DATABASE),postgres) + test_submodules := $(filter-out %/riverdriver/rivermysql %/riverdriver/riversqlite,$(submodules)) + driver_test_flags := -skip '$(mysql_test_pattern)|$(sqlite_test_pattern)' + legacy_driver_test_flags += -skip '$(mysql_test_pattern)|$(sqlite_test_pattern)' else ifeq ($(TEST_DATABASE),sqlite) test_submodules := $(filter %/riverdriver/riverdrivertest %/riverdriver/riversqlite,$(submodules)) driver_test_flags := -run '$(sqlite_test_pattern)' legacy_driver_test_flags := -run '$(sqlite_test_pattern)/WithTx$$' else ifneq ($(TEST_DATABASE),all) - $(error TEST_DATABASE must be all, postgres, or sqlite) + $(error TEST_DATABASE must be all, mysql, postgres, or sqlite) endif # Definitions of following tasks look ugly, but they're done this way because to @@ -112,6 +119,9 @@ lint/rust: ## Run Rust formatting and clippy checks, including single-backend bu cd rust && cargo fmt --all -- --check cd rust && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features postgres --all-targets --locked -- -D warnings + cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features mysql --all-targets --locked -- -D warnings + # Test dependencies enable other backends; also check MySQL libraries on their own. + cd rust && cargo clippy -p riverqueue -p riverqueue-migrate --lib --no-default-features --features mysql --locked -- -D warnings cd rust && cargo clippy -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features sqlite --all-targets --locked -- -D warnings cd rust && $(RUST_POSTGRES_TESTS_ENV) cargo clippy -p riverqueue -p riverqueue-migrate --all-targets --all-features --locked -- -D warnings @@ -148,7 +158,7 @@ lint/js: build/js pnpm -C js run typecheck:next .PHONY: test -test:: ## Run tests (TEST_DATABASE=all, postgres, or sqlite) +test:: ## Run tests (TEST_DATABASE=all, mysql, postgres, or sqlite) define test-target test:: ; cd $1 && go test ./... -timeout 2m $(if $(filter %/riverdriver/riverdrivertest,$1),$(driver_test_flags)) endef @@ -156,7 +166,7 @@ $(foreach mod,$(test_submodules),$(eval $(call test-target,$(mod)))) # Exercise the temporary savepoint fallback as well as default transaction reuse. test:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . $(legacy_driver_test_flags) -timeout 2m -ifneq ($(TEST_DATABASE),sqlite) +ifneq ($(filter all postgres,$(TEST_DATABASE)),) test:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -run '^TestDriverRiverPgxV5$$/.*/WithTx$$' -timeout 2m endif @@ -173,7 +183,7 @@ RUST_POSTGRES_TESTS_ENV = RUSTFLAGS="$$RUSTFLAGS --cfg river_postgres_tests" \ test/java: ## Run Java tests, executable CLI tests, and formatting checks test/java: generate/fixtures verify/java-migrations test/java: test/java/tools - $(java_mvn) -f java/pom.xml verify + $(java_mvn) -f java/pom.xml verify -DexcludedGroups=mysql,postgres # Fixture comparisons use temporary SQLite databases, with no Postgres or legacy adapter. .PHONY: test/java/conformance @@ -181,16 +191,26 @@ test/java/conformance: ## Run Java tests that check Go-generated conformance fix test/java/conformance: generate/fixtures $(java_mvn) -f java/pom.xml -pl river test -Dgroups=conformance -Driver.test.database=sqlite +.PHONY: test/java/conformance/mysql +test/java/conformance/mysql: generate/fixtures ## Run Java's Go fixture checks on MySQL (requires RIVER_TEST_DATABASE_URL) + @test -n "$$RIVER_TEST_DATABASE_URL" || { echo "RIVER_TEST_DATABASE_URL is required" >&2; exit 1; } + $(java_mvn) -f java/pom.xml -pl river test -Dgroups=conformance -Driver.test.database=mysql + +.PHONY: test/java/mysql +test/java/mysql: generate/fixtures verify/java-migrations ## Run Java tests against MySQL (requires RIVER_TEST_DATABASE_URL) + @test -n "$$RIVER_TEST_DATABASE_URL" || { echo "RIVER_TEST_DATABASE_URL is required" >&2; exit 1; } + $(java_mvn) -f java/pom.xml verify -Driver.test.database=mysql -DexcludedGroups=postgres + .PHONY: test/java/postgres test/java/postgres: ## Run Java tests with Postgres client and worker coverage (requires RIVER_TEST_DATABASE_URL) test/java/postgres: generate/fixtures verify/java-migrations @test -n "$$RIVER_TEST_DATABASE_URL" || { echo "RIVER_TEST_DATABASE_URL is required" >&2; exit 1; } - $(java_mvn) -f java/pom.xml verify -Driver.test.database=postgres + $(java_mvn) -f java/pom.xml verify -Driver.test.database=postgres -DexcludedGroups=mysql .PHONY: test/java/sqlite test/java/sqlite: ## Run Java unit, SQLite, and executable CLI tests without Postgres test/java/sqlite: generate/fixtures verify/java-migrations - $(java_mvn) -f java/pom.xml verify -Driver.test.database=sqlite -DexcludedGroups=postgres + $(java_mvn) -f java/pom.xml verify -Driver.test.database=sqlite -DexcludedGroups=mysql,postgres .PHONY: test/java/tools test/java/tools: ## Test Java maintenance tools, release tooling, and Maven warning enforcement @@ -216,9 +236,19 @@ test/js/conformance: ## Run JavaScript tests that check Go-generated conformance test/js/conformance: generate/fixtures pnpm -C js exec vitest run src/conformance.test.ts src/cron.test.ts src/runtime/completion-command.test.ts src/runtime/notification-pump.conformance.test.ts +.PHONY: test/js/conformance/mysql +test/js/conformance/mysql: generate/fixtures ## Run JavaScript's Go fixture checks on MySQL (requires TEST_MYSQL_URL) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + pnpm -C js exec vitest run --config vitest.integration.config.ts driver/mysql/src/driver.integration.test.ts -t 'Go conformance' + # Integration tests use TEST_DATABASE_URL (default # postgres://localhost:5432/river_test), migrated with # `node js/cli/dist/bin.js migrate-up`. +.PHONY: test/js/mysql +test/js/mysql: build/js generate/fixtures ## Run JavaScript MySQL integration tests (requires TEST_MYSQL_URL) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + pnpm -C js exec vitest run --config vitest.integration.config.ts driver/mysql/src/driver.integration.test.ts + .PHONY: test/js/integration test/js/integration: ## Run JavaScript integration tests against Postgres test/js/integration: build/js @@ -244,6 +274,17 @@ test/rust/conformance: ## Run Rust tests that check Go-generated conformance fix test/rust/conformance: generate/fixtures cd rust && cargo test -p riverqueue --features tzdb,sqlite --lib --test protocol_fixtures --test resumable_conformance --locked +.PHONY: test/rust/conformance/mysql +test/rust/conformance/mysql: generate/fixtures ## Run Rust's Go fixture checks on MySQL (requires RIVER_RUST_MYSQL_URL) + @test -n "$$RIVER_RUST_MYSQL_URL" || { echo "RIVER_RUST_MYSQL_URL is required" >&2; exit 1; } + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo test -p riverqueue --all-features --lib database::mysql::tests::conformance --locked + +.PHONY: test/rust/mysql +test/rust/mysql: generate/fixtures verify/rust-migrations ## Run Rust MySQL integration tests (requires RIVER_RUST_MYSQL_URL) + @test -n "$$RIVER_RUST_MYSQL_URL" || { echo "RIVER_RUST_MYSQL_URL is required" >&2; exit 1; } + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo test -p riverqueue --all-features --lib database::mysql::tests --locked + cd rust && RUSTFLAGS="$$RUSTFLAGS --cfg river_mysql_tests" cargo clippy -p riverqueue --all-features --tests --locked -- -D warnings + .PHONY: test/rust/postgres test/rust/postgres: ## Run all Rust tests, including Postgres integration tests (requires RIVER_RUST_DATABASE_URL) test/rust/postgres: generate/fixtures @@ -267,6 +308,7 @@ doc/rust: ## Build Rust API documentation, compiled examples, and doctests for e cd rust && RUSTDOCFLAGS="-D warnings" cargo doc --workspace --all-features --no-deps --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test --workspace --all-features --doc --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features postgres --doc --locked + cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features mysql --doc --locked cd rust && RUSTDOCFLAGS="-D warnings" cargo test -p riverqueue -p riverqueue-migrate -p riverqueue-cli -p riverqueue-test --no-default-features --features sqlite --doc --locked cd rust && cargo check --workspace --examples --all-features --locked @@ -286,7 +328,7 @@ check/java/github-release: ## Build, test, and stage all Java GitHub release ass .PHONY: check/java/release check/java/release: ## Build a signed Maven Central bundle without uploading (requires an OpenPGP key) check/java/release: generate/fixtures verify/java-migrations - $(java_mvn) -f java/pom.xml -Prelease clean deploy -Driver.test.database=sqlite -DexcludedGroups=postgres -Dmaven.install.skip=true -Dcentral.skipUpload=true + $(java_mvn) -f java/pom.xml -Prelease clean deploy -Driver.test.database=sqlite -DexcludedGroups=mysql,postgres -Dmaven.install.skip=true -Dcentral.skipUpload=true .PHONY: check/js/dependencies check/js/dependencies: ## Audit JavaScript advisories and production dependency licenses @@ -339,14 +381,14 @@ check/rust/semver: ## Check Rust APIs against RUST_SEMVER_BASELINE_REV (default: fi .PHONY: test/race -test/race:: ## Run tests with race detector (TEST_DATABASE=all, postgres, or sqlite) +test/race:: ## Run tests with race detector (TEST_DATABASE=all, mysql, postgres, or sqlite) define test-race-target test/race:: ; cd $1 && go test ./... -race -timeout 2m $(if $(filter %/riverdriver/riverdrivertest,$1),$(driver_test_flags)) endef $(foreach mod,$(test_submodules),$(eval $(call test-race-target,$(mod)))) test/race:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -race $(legacy_driver_test_flags) -timeout 2m -ifneq ($(TEST_DATABASE),sqlite) +ifneq ($(filter all postgres,$(TEST_DATABASE)),) test/race:: ; cd ./riverdriver/riverdrivertest && RIVER_USE_LEGACY_SUBTRANSACTIONS=1 go test . -race -run '^TestDriverRiverPgxV5$$/.*/WithTx$$' -timeout 2m endif @@ -357,6 +399,10 @@ define bench-target endef $(foreach mod,$(submodules),$(eval $(call bench-target,$(mod)))) +.PHONY: bench/compare +bench/compare: ## Compare Go driver throughput on MySQL, Postgres, and SQLite (BENCH_COMPARE_ARGS="-jobs 30000 -samples 3") + cd cmd/river && go run ./benchcompare $(BENCH_COMPARE_ARGS) + .PHONY: bench/rust bench/rust: ## Run the destructive Rust Postgres throughput benchmark cd rust && cargo run --release --locked -p riverqueue-cli --bin riverqueue -- bench $(if $(DATABASE_URL),--database-url "$(DATABASE_URL)") $(RUST_BENCH_ARGS) @@ -404,6 +450,7 @@ verify/rust-migrations: ## Verify Rust migrations match the canonical migrations .PHONY: verify/sqlc verify/sqlc: ## Verify generated sqlc cd riverdriver/riverdatabasesql/internal/dbsqlc && $(SQLC) diff + cd riverdriver/rivermysql/internal/dbsqlc && $(SQLC) diff cd riverdriver/riverpgxv5/internal/dbsqlc && $(SQLC) diff cd riverdriver/riversqlite/internal/dbsqlc && $(SQLC) diff @@ -422,6 +469,11 @@ test/ruby: ## Run Ruby core, Active Record, Sequel, and Rails tests test/ruby/conformance: ## Run Ruby tests that check Go-generated conformance fixtures $(MAKE) -C ruby test/conformance +.PHONY: test/ruby/conformance/mysql +test/ruby/conformance/mysql: ## Run Ruby's Go fixture checks on MySQL (requires TEST_MYSQL_URL with mysql2://) + @test -n "$$TEST_MYSQL_URL" || { echo "TEST_MYSQL_URL is required" >&2; exit 1; } + TEST_DATABASE=mysql $(MAKE) -C ruby test/conformance + typecheck/ruby: ## Check Ruby types with Steep $(MAKE) -C ruby type-check diff --git a/client.go b/client.go index 0b64c561f..522948b7c 100644 --- a/client.go +++ b/client.go @@ -1067,11 +1067,11 @@ func NewClient[TTx any](driver riverdriver.Driver[TTx], config *Config) (*Client client.testSignals.queueCleaner = &queueCleaner.TestSignals } - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - sqliteNotificationCleaner := maintenance.NewSQLiteNotificationCleaner(archetype, &maintenance.SQLiteNotificationCleanerConfig{ + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + notificationCleaner := maintenance.NewSQLiteNotificationCleaner(archetype, &maintenance.SQLiteNotificationCleanerConfig{ Schema: config.Schema, }, driver.GetExecutor()) - maintenanceServices = append(maintenanceServices, sqliteNotificationCleaner) + maintenanceServices = append(maintenanceServices, notificationCleaner) } { @@ -2576,11 +2576,17 @@ func (c *Client[TTx]) JobList(ctx context.Context, params *JobListParams) (*JobL if params == nil { params = NewJobListParams() } - params.schema = c.config.Schema - if c.driver.DatabaseName() == riverdriver.DatabaseNameSQLite && params.metadataCalled { return nil, errJobListParamsMetadataNotSupportedSQLite } + if params.metadataCalled { + var err error + params, err = params.withMetadataPredicateSQL(c.driver.SQLFragmentColumnContainsJSON) + if err != nil { + return nil, err + } + } + params.schema = c.config.Schema dbParams, err := params.toDBParams() if err != nil { @@ -2617,11 +2623,17 @@ func (c *Client[TTx]) JobListTx(ctx context.Context, tx TTx, params *JobListPara if params == nil { params = NewJobListParams() } - params.schema = c.config.Schema - if c.driver.DatabaseName() == riverdriver.DatabaseNameSQLite && params.metadataCalled { return nil, errJobListParamsMetadataNotSupportedSQLite } + if params.metadataCalled { + var err error + params, err = params.withMetadataPredicateSQL(c.driver.SQLFragmentColumnContainsJSON) + if err != nil { + return nil, err + } + } + params.schema = c.config.Schema dbParams, err := params.toDBParams() if err != nil { diff --git a/client_test.go b/client_test.go index 271d04461..678817e0f 100644 --- a/client_test.go +++ b/client_test.go @@ -9069,6 +9069,32 @@ func Test_NewClient_Overrides(t *testing.T) { require.Len(t, client.config.WorkerMiddleware, 1) } +func Test_NewClient_MySQLNotificationCleaner(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + var ( + dbPool = riversharedtest.DBPool(ctx, t) + pgxDriver = riverpgxv5.New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, pgxDriver, nil) + ) + + workers := NewWorkers() + AddWorker(workers, &noOpWorker{}) + + client, err := NewClient(NewDriverMySQLDatabaseName(dbPool), &Config{ + Queues: map[string]QueueConfig{QueueDefault: {MaxWorkers: 1}}, + Schema: schema, + TestOnly: true, + Workers: workers, + }) + require.NoError(t, err) + + notificationCleaner := maintenance.GetService[*maintenance.SQLiteNotificationCleaner](client.queueMaintainer) + require.Equal(t, schema, notificationCleaner.Config.Schema) +} + func Test_NewClient_PluginsAndHybrids(t *testing.T) { t.Parallel() @@ -10600,6 +10626,23 @@ func TestDefaultClientIDWithHost(t *testing.T) { require.Equal(t, strings.Repeat("a", 60)+"_2024_03_07T04_39_12_123456", defaultClientIDWithHost(startedAt, strings.Repeat("a", 61))) } +// DriverMySQLDatabaseName is a pgx driver that reports MySQL as its database +// name so MySQL-specific client wiring can be tested without importing the +// separate MySQL driver module. +type DriverMySQLDatabaseName struct { + riverpgxv5.Driver +} + +// NewDriverMySQLDatabaseName returns a new test driver that reports MySQL as +// its database name. +func NewDriverMySQLDatabaseName(dbPool *pgxpool.Pool) *DriverMySQLDatabaseName { + return &DriverMySQLDatabaseName{ + Driver: *riverpgxv5.New(dbPool), + } +} + +func (d *DriverMySQLDatabaseName) DatabaseName() string { return riverdriver.DatabaseNameMySQL } + // DriverPollOnly simulates a driver without a listener. An example of this is // Postgres through `riverdatabasesql`, which is Postgres (so it can notify), // but where `database/sql` provides no listener mechanism. We could use the diff --git a/cmd/river/benchcompare/database.go b/cmd/river/benchcompare/database.go new file mode 100644 index 000000000..4e4e80e9b --- /dev/null +++ b/cmd/river/benchcompare/database.go @@ -0,0 +1,168 @@ +package main + +import ( + "cmp" + "context" + "crypto/rand" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + _ "modernc.org/sqlite" + + "github.com/riverqueue/river/riverdriver/rivermysql" + "github.com/riverqueue/river/riverdriver/riverpgxv5" + "github.com/riverqueue/river/riverdriver/riversqlite" +) + +func runDatabase(ctx context.Context, backend string, cfg config, sample *trial) error { + // Only drop objects successfully created by this invocation. Never truncate + // a caller's tables, even when using a shared local development server. + name := "river_bench_" + strings.ToLower(rand.Text()) + switch backend { + case "mysql": + return runMySQL(ctx, cfg, name, sample) + case "postgres": + return runPostgres(ctx, cfg, name, sample) + case "sqlite": + return runSQLite(ctx, cfg, sample) + default: + return fmt.Errorf("unknown backend: %s", backend) + } +} + +func runMySQL(ctx context.Context, cfg config, name string, sample *trial) (err error) { + connConfig, err := mysql.ParseDSN(cmp.Or(os.Getenv("RIVER_BENCH_MYSQL_DSN"), "root@tcp(127.0.0.1:3306)/")) + if err != nil { + return err + } + connConfig.DBName = "" + connConfig.Loc = time.UTC + connConfig.MultiStatements = true + connConfig.ParseTime = true + if connConfig.Params == nil { + connConfig.Params = make(map[string]string) + } + connConfig.Params["time_zone"] = "'+00:00'" + connector, err := mysql.NewConnector(connConfig) + if err != nil { + return err + } + admin := sql.OpenDB(connector) + defer func() { err = errors.Join(err, admin.Close()) }() + if _, err := admin.ExecContext(ctx, "CREATE DATABASE `"+name+"` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"); err != nil { + return err + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, dropErr := admin.ExecContext(cleanupCtx, "DROP DATABASE `"+name+"`") + err = errors.Join(err, dropErr) + }() + connConfig.DBName = name + connector, err = mysql.NewConnector(connConfig) + if err != nil { + return err + } + pool := sql.OpenDB(connector) + defer func() { err = errors.Join(err, pool.Close()) }() + pool.SetMaxOpenConns(cfg.Connections) + pool.SetMaxIdleConns(cfg.Connections) + sample.Settings, err = readSettings(pool.QueryRowContext(ctx, + "SELECT VERSION(), @@innodb_flush_log_at_trx_commit, @@sync_binlog, @@log_bin, @@innodb_buffer_pool_size"), + "version", "innodb_flush_log_at_trx_commit", "sync_binlog", "log_bin", "innodb_buffer_pool_size") + if err != nil { + return err + } + sample.Settings["interpolate_params"] = strconv.FormatBool(connConfig.InterpolateParams) + return measure(ctx, rivermysql.New(pool), "", cfg, sample) +} + +func runPostgres(ctx context.Context, cfg config, name string, sample *trial) (err error) { + url := cmp.Or(os.Getenv("RIVER_BENCH_POSTGRES_URL"), "postgres://localhost/postgres") + admin, err := pgx.Connect(ctx, url) + if err != nil { + return err + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + err = errors.Join(err, admin.Close(cleanupCtx)) + }() + identifier := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+identifier); err != nil { + return err + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, dropErr := admin.Exec(cleanupCtx, "DROP SCHEMA "+identifier+" CASCADE") + err = errors.Join(err, dropErr) + }() + poolConfig, err := pgxpool.ParseConfig(url) + if err != nil { + return err + } + poolConfig.MaxConns = int32(cfg.Connections) //nolint:gosec // Small, user-selected benchmark pool size. + poolConfig.ConnConfig.RuntimeParams["search_path"] = name + pool, err := pgxpool.NewWithConfig(ctx, poolConfig) + if err != nil { + return err + } + defer pool.Close() + sample.Settings, err = readSettings(pool.QueryRow(ctx, + "SELECT current_setting('server_version'), current_setting('fsync'), current_setting('synchronous_commit'), current_setting('full_page_writes'), current_setting('shared_buffers')"), + "version", "fsync", "synchronous_commit", "full_page_writes", "shared_buffers") + if err != nil { + return err + } + return measure(ctx, riverpgxv5.New(pool), name, cfg, sample) +} + +func runSQLite(ctx context.Context, cfg config, sample *trial) (err error) { + dir, err := os.MkdirTemp("", "river-bench-sqlite-") + if err != nil { + return err + } + defer func() { err = errors.Join(err, os.RemoveAll(dir)) }() + pool, err := sql.Open("sqlite", filepath.Join(dir, "bench.sqlite3")) + if err != nil { + return err + } + defer func() { err = errors.Join(err, pool.Close()) }() + pool.SetMaxOpenConns(1) + if _, err := pool.ExecContext(ctx, "PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA busy_timeout = 5000;"); err != nil { + return err + } + sample.Settings, err = readSettings(pool.QueryRowContext(ctx, + "SELECT sqlite_version(), (SELECT journal_mode FROM pragma_journal_mode), (SELECT synchronous FROM pragma_synchronous), (SELECT cache_size FROM pragma_cache_size), (SELECT page_size FROM pragma_page_size), (SELECT fullfsync FROM pragma_fullfsync)"), + "version", "journal_mode", "synchronous", "cache_size", "page_size", "fullfsync") + if err != nil { + return err + } + return measure(ctx, riversqlite.New(pool), "", cfg, sample) +} + +func readSettings(row interface{ Scan(dest ...any) error }, names ...string) (map[string]string, error) { + values := make([]string, len(names)) + args := make([]any, len(names)) + for i := range values { + args[i] = &values[i] + } + if err := row.Scan(args...); err != nil { + return nil, err + } + settings := make(map[string]string, len(names)) + for i, name := range names { + settings[name] = values[i] + } + return settings, nil +} diff --git a/cmd/river/benchcompare/main.go b/cmd/river/benchcompare/main.go new file mode 100644 index 000000000..71c57c36f --- /dev/null +++ b/cmd/river/benchcompare/main.go @@ -0,0 +1,195 @@ +// Command benchcompare compares River's Go drivers on isolated databases. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "runtime/debug" + "slices" + "strconv" + "strings" + "syscall" + "time" +) + +type config struct { + Backends []string `json:"backends"` + BatchSize int `json:"batch_size"` + Connections int `json:"connections"` + GoMaxProcs int `json:"gomaxprocs"` + InsertMode string `json:"insert_mode"` + Jobs int `json:"jobs"` + PayloadBytes int `json:"payload_bytes"` + Samples int `json:"samples"` + Timeout time.Duration `json:"timeout_ns"` + Warmups int `json:"warmups"` + Workers int `json:"workers"` +} + +type report struct { + Build string `json:"build"` + Config config `json:"config"` + Error string `json:"error,omitempty"` + Machine map[string]string `json:"machine"` + StartedAt time.Time `json:"started_at"` + Trials []trial `json:"trials"` +} + +type trial struct { + Backend string `json:"backend"` + Completed int `json:"completed"` + DrainJobsPerSecond float64 `json:"drain_jobs_per_second"` + DrainSeconds float64 `json:"drain_seconds"` + InsertJobsPerSecond float64 `json:"insert_jobs_per_second"` + InsertSeconds float64 `json:"insert_seconds"` + LoadAfter string `json:"load_after"` + LoadBefore string `json:"load_before"` + Round int `json:"round"` + Settings map[string]string `json:"settings"` + StartedAt time.Time `json:"started_at"` + Warmup bool `json:"warmup"` +} + +func main() { + var cfg config + backends := flag.String("backends", "mysql,postgres,sqlite", "comma-separated databases to benchmark") + flag.IntVar(&cfg.BatchSize, "batch-size", 1000, "jobs per InsertMany call") + flag.IntVar(&cfg.Connections, "connections", 10, "Postgres/MySQL pool limit (SQLite always uses one)") + flag.IntVar(&cfg.GoMaxProcs, "gomaxprocs", 4, "Go CPU parallelism; database servers retain their own settings") + flag.StringVar(&cfg.InsertMode, "insert-mode", "returning", "insertion API: returning (InsertMany) or fast (InsertManyFast, count only)") + flag.IntVar(&cfg.Jobs, "jobs", 30000, "jobs inserted and completed per trial") + flag.IntVar(&cfg.PayloadBytes, "payload-bytes", 256, "payload string length, excluding the JSON envelope") + flag.IntVar(&cfg.Samples, "samples", 3, "measured rounds, each containing all selected databases") + flag.DurationVar(&cfg.Timeout, "timeout", 5*time.Minute, "maximum time per database trial, including setup") + flag.IntVar(&cfg.Warmups, "warmups", 1, "unmeasured rounds before collecting samples") + flag.IntVar(&cfg.Workers, "workers", 100, "concurrent no-op workers") + output := flag.String("output", filepath.Join(os.TempDir(), "river-bench-"+time.Now().UTC().Format("20060102T150405.000000000Z")+".json"), "new JSON results file (never overwrites an existing file)") + flag.Parse() + cfg.Backends = strings.Split(*backends, ",") + + if err := run(cfg, *output); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(cfg config, output string) error { + if len(cfg.Backends) == 0 { + return errors.New("select at least one backend") + } + for i, backend := range cfg.Backends { + if backend != "mysql" && backend != "postgres" && backend != "sqlite" { + return fmt.Errorf("unknown backend %q (use mysql, postgres, or sqlite)", backend) + } + if slices.Contains(cfg.Backends[:i], backend) { + return fmt.Errorf("duplicate backend %q", backend) + } + } + if cfg.InsertMode != "fast" && cfg.InsertMode != "returning" { + return errors.New("insert-mode must be fast or returning") + } + if cfg.BatchSize < 1 || cfg.Connections < 2 || cfg.GoMaxProcs < 1 || cfg.Jobs < 1 || cfg.PayloadBytes < 0 || cfg.Samples < 1 || cfg.Timeout <= 0 || cfg.Warmups < 0 || cfg.Workers < 1 { + return errors.New("counts and timeout must be positive; connections >= 2; warmups and payload bytes may be zero") + } + // Check before creating any database objects. Rewrites below update only the + // file owned by this invocation, preserving partial results on failure. + file, err := os.OpenFile(output, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } + runtime.GOMAXPROCS(cfg.GoMaxProcs) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + result := report{ + Config: cfg, + Machine: map[string]string{ + "arch": runtime.GOARCH, "os": runtime.GOOS, "go": runtime.Version(), + "cpu_count": strconv.Itoa(runtime.NumCPU()), "uname": capture("uname", "-a"), + "cpu": capture("sysctl", "-n", "machdep.cpu.brand_string"), + "memory_bytes": capture("sysctl", "-n", "hw.memsize"), + "git_commit": capture("git", "rev-parse", "HEAD"), + "git_status": capture("git", "status", "--short"), + }, + StartedAt: time.Now().UTC(), + } + if info, ok := debug.ReadBuildInfo(); ok { + result.Build = info.String() + } + fmt.Printf("Results: %s\n", output) + backends := cfg.Backends + for round := range cfg.Warmups + cfg.Samples { + // Rotate order to distribute changing machine load across backends. + for offset := range backends { + backend := backends[(round+offset)%len(backends)] + sample := trial{Backend: backend, Round: round + 1, StartedAt: time.Now().UTC(), Warmup: round < cfg.Warmups} + sample.LoadBefore = capture("uptime") + fmt.Printf("round %d %s (warmup=%t)\n", sample.Round, backend, sample.Warmup) + trialCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) + err := runDatabase(trialCtx, backend, cfg, &sample) + cancel() + sample.LoadAfter = capture("uptime") + result.Trials = append(result.Trials, sample) + if err != nil { + result.Error = fmt.Sprintf("%s round %d: %v", backend, sample.Round, err) + } + if saveErr := saveReport(output, result); saveErr != nil { + return errors.Join(err, saveErr) + } + if err != nil { + return errors.New(result.Error) + } + fmt.Printf(" insert %.0f jobs/s; drain %.0f jobs/s; verified %d completions\n", sample.InsertJobsPerSecond, sample.DrainJobsPerSecond, sample.Completed) + } + } + fmt.Println("\nBackend Insert jobs/s (median [min, max]) Drain jobs/s (median [min, max])") + for _, backend := range backends { + var inserts, drains []float64 + for _, sample := range result.Trials { + if sample.Backend == backend && !sample.Warmup { + inserts = append(inserts, sample.InsertJobsPerSecond) + drains = append(drains, sample.DrainJobsPerSecond) + } + } + fmt.Printf("%-10s %-37s %s\n", backend, summarize(inserts), summarize(drains)) + } + return nil +} + +func capture(name string, args ...string) string { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).Output() + if err != nil { + return "unavailable" + } + return strings.TrimSpace(string(out)) +} + +func saveReport(path string, result report) error { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o600) +} + +func summarize(values []float64) string { + slices.Sort(values) + middle := len(values) / 2 + median := values[middle] + if len(values)%2 == 0 { + median = (values[middle-1] + values[middle]) / 2 + } + return fmt.Sprintf("%.0f [%.0f, %.0f]", median, values[0], values[len(values)-1]) +} diff --git a/cmd/river/benchcompare/measure.go b/cmd/river/benchcompare/measure.go new file mode 100644 index 000000000..ab68b1f3b --- /dev/null +++ b/cmd/river/benchcompare/measure.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/riverqueue/river" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivermigrate" + "github.com/riverqueue/river/rivertype" +) + +type benchArgs struct { + Payload string `json:"payload"` +} + +func (benchArgs) Kind() string { return "benchcompare" } + +type benchWorker struct { + river.WorkerDefaults[benchArgs] +} + +func (*benchWorker) Work(context.Context, *river.Job[benchArgs]) error { return nil } + +func measure[TTx any](ctx context.Context, driver riverdriver.Driver[TTx], schema string, cfg config, sample *trial) (err error) { + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + migrator, err := rivermigrate.New(driver, &rivermigrate.Config{Logger: logger, Schema: schema}) + if err != nil { + return err + } + if _, err := migrator.Migrate(ctx, rivermigrate.DirectionUp, nil); err != nil { + return err + } + workers := river.NewWorkers() + river.AddWorker(workers, &benchWorker{}) + client, err := river.NewClient(driver, &river.Config{ + CompletedJobRetentionPeriod: -1, + FetchCooldown: 2 * time.Millisecond, + FetchPollInterval: 20 * time.Millisecond, + Logger: logger, + Queues: map[string]river.QueueConfig{river.QueueDefault: {MaxWorkers: cfg.Workers}}, + Schema: schema, + Workers: workers, + }) + if err != nil { + return err + } + events, cancel := client.SubscribeConfig(&river.SubscribeConfig{ + ChanSize: cfg.Jobs, + Kinds: []river.EventKind{river.EventKindJobCompleted, river.EventKindJobFailed, river.EventKindJobCancelled}, + }) + defer cancel() + params := make([]river.InsertManyParams, min(cfg.BatchSize, cfg.Jobs)) + for i := range params { + params[i].Args = benchArgs{Payload: strings.Repeat("x", cfg.PayloadBytes)} + } + start := time.Now() + for inserted := 0; inserted < cfg.Jobs; { + batch := params[:min(len(params), cfg.Jobs-inserted)] + var count int + if cfg.InsertMode == "fast" { + count, err = client.InsertManyFast(ctx, batch) + } else { + var rows []*rivertype.JobInsertResult + rows, err = client.InsertMany(ctx, batch) + count = len(rows) + } + if err != nil { + return fmt.Errorf("insert: %w", err) + } + if count != len(batch) { + return fmt.Errorf("insert returned %d jobs, expected %d", count, len(batch)) + } + inserted += count + } + sample.InsertSeconds = time.Since(start).Seconds() + sample.InsertJobsPerSecond = float64(cfg.Jobs) / sample.InsertSeconds + + // Include client startup in drain time. Count completion events after the + // database acknowledgement, rather than counting calls to the no-op worker. + start = time.Now() + if err := client.Start(ctx); err != nil { + return err + } + defer func() { + stopCtx, stopCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer stopCancel() + err = errors.Join(err, client.StopAndCancel(stopCtx)) + }() + for sample.Completed < cfg.Jobs { + select { + case <-ctx.Done(): + return ctx.Err() + case event, ok := <-events: + if !ok { + return errors.New("completion subscription closed early") + } + if event.Kind != river.EventKindJobCompleted || event.Job.Attempt != 1 { + return fmt.Errorf("unexpected job outcome: %s, attempt %d", event.Kind, event.Job.Attempt) + } + sample.Completed++ + } + } + sample.DrainSeconds = time.Since(start).Seconds() + sample.DrainJobsPerSecond = float64(cfg.Jobs) / sample.DrainSeconds + if err := client.Stop(ctx); err != nil { + return err + } + completed, err := driver.GetExecutor().JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateCompleted, + }) + if err != nil { + return err + } + if completed != cfg.Jobs { + return fmt.Errorf("database has %d completed jobs, expected %d", completed, cfg.Jobs) + } + return nil +} diff --git a/cmd/river/benchcompare/measure_test.go b/cmd/river/benchcompare/measure_test.go new file mode 100644 index 000000000..e092a303a --- /dev/null +++ b/cmd/river/benchcompare/measure_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRunSQLite(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"fast", "returning"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + + // Exercise a partial final insertion batch and the real completion path, + // including verification of durable rows and cleanup of the temporary DB. + cfg := config{BatchSize: 10, InsertMode: mode, Jobs: 23, PayloadBytes: 32, Workers: 2} + var sample trial + require.NoError(t, runSQLite(t.Context(), cfg, &sample)) + require.Equal(t, cfg.Jobs, sample.Completed) + require.Positive(t, sample.DrainJobsPerSecond) + require.Positive(t, sample.InsertJobsPerSecond) + require.Equal(t, "wal", sample.Settings["journal_mode"]) + require.Equal(t, "2", sample.Settings["synchronous"]) + require.Less(t, sample.DrainSeconds, (30 * time.Second).Seconds()) + }) + } +} + +func TestSummarize(t *testing.T) { + t.Parallel() + + t.Run("Even", func(t *testing.T) { + t.Parallel() + + require.Equal(t, "15 [10, 20]", summarize([]float64{20, 10})) + }) + + t.Run("Odd", func(t *testing.T) { + t.Parallel() + + require.Equal(t, "20 [10, 40]", summarize([]float64{40, 10, 20})) + }) +} diff --git a/cmd/river/go.mod b/cmd/river/go.mod index 8896e2c98..2d52a9df2 100644 --- a/cmd/river/go.mod +++ b/cmd/river/go.mod @@ -5,10 +5,12 @@ go 1.26.0 toolchain go1.26.6 require ( + github.com/go-sql-driver/mysql v1.10.0 github.com/jackc/pgx/v5 v5.11.0 github.com/lmittmann/tint v1.2.1 github.com/riverqueue/river v0.51.1 github.com/riverqueue/river/riverdriver v0.51.1 + github.com/riverqueue/river/riverdriver/rivermysql v0.0.0-00010101000000-000000000000 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 github.com/riverqueue/river/riverdriver/riversqlite v0.51.1 github.com/riverqueue/river/rivershared v0.51.1 @@ -19,6 +21,7 @@ require ( ) require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/dustin/go-humanize v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -42,3 +45,7 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) + +// Remove this replacement after rivermysql has had its first coordinated +// release with the other Go modules. +replace github.com/riverqueue/river/riverdriver/rivermysql => ../../riverdriver/rivermysql diff --git a/cmd/river/go.sum b/cmd/river/go.sum index 8625152b7..9530c081f 100644 --- a/cmd/river/go.sum +++ b/cmd/river/go.sum @@ -1,7 +1,11 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -29,18 +33,18 @@ github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2K github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/riverdriver/riversqlite v0.51.0 h1:mWm7sg8L2U/cF2CdgQSOC1kX90S9sWomMZZgwHPD1Gs= -github.com/riverqueue/river/riverdriver/riversqlite v0.51.0/go.mod h1:akyTyxi/+4W0ND9QJzKOtxq880T0LFoRI5KlKILmdWs= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/riverdriver/riversqlite v0.51.1 h1:VmAPT1AETH3sqbbHG9GPiH0OmQFXJk+hFeRgCVUKoI0= +github.com/riverqueue/river/riverdriver/riversqlite v0.51.1/go.mod h1:G6NSbOzZO4eJwvtIHzwh9i4tAzKwoUrJ25AzkZpFwFY= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/cmd/river/rivercli/command.go b/cmd/river/rivercli/command.go index 317e96075..b4ac9eaf1 100644 --- a/cmd/river/rivercli/command.go +++ b/cmd/river/rivercli/command.go @@ -7,11 +7,14 @@ import ( "fmt" "io" "log/slog" + "net" + "net/url" "os" "strconv" "strings" "time" + "github.com/go-sql-driver/mysql" "github.com/jackc/pgx/v5/pgxpool" _ "modernc.org/sqlite" ) @@ -80,6 +83,19 @@ func RunCommand[TOpts CommandOpts](ctx context.Context, bundle *RunCommandBundle driverProcurer := bundle.DriverProcurer if databaseURL != nil { switch protocol { + case "mysql": + dbPool, err := openMySQLPool(*databaseURL) + if err != nil { + return false, err + } + defer dbPool.Close() + + if procurer, ok := driverProcurer.(DriverProcurerMySQL); ok { + procurer.InitMySQL(dbPool) + } else { + driverProcurer = &mysqlDriverProcurer{dbPool: dbPool} + } + case "postgres", "postgresql": dbPool, err := openPgxV5DBPool(ctx, *databaseURL, bundle.StatementTimeout) if err != nil { @@ -109,7 +125,7 @@ func RunCommand[TOpts CommandOpts](ctx context.Context, bundle *RunCommandBundle } default: - return false, fmt.Errorf("unsupported database URL (`%s`); try one with a `postgres://`, `postgresql://`, or `sqlite://` scheme/prefix", *bundle.DatabaseURL) + return false, fmt.Errorf("unsupported database URL (`%s`); try one with a `mysql://`, `postgres://`, `postgresql://`, or `sqlite://` scheme/prefix", *bundle.DatabaseURL) } } @@ -133,6 +149,18 @@ func RunCommand[TOpts CommandOpts](ctx context.Context, bundle *RunCommandBundle return nil } +func openMySQLPool(databaseURL string) (*sql.DB, error) { + config, err := parseMySQLURL(databaseURL) + if err != nil { + return nil, err + } + connector, err := mysql.NewConnector(config) + if err != nil { + return nil, fmt.Errorf("error configuring MySQL connection: %w", err) + } + return sql.OpenDB(connector), nil +} + func openPgxV5DBPool(ctx context.Context, databaseURL string, statementTimeout *time.Duration) (*pgxpool.Pool, error) { const ( defaultIdleInTransactionSessionTimeout = 11 * time.Second // should be greater than statement timeout because statements count towards idle-in-transaction @@ -193,6 +221,35 @@ func openSQLitePool(protocol, urlWithoutProtocol string) (*sql.DB, error) { return dbPool, nil } +func parseMySQLURL(databaseURL string) (*mysql.Config, error) { + parsed, err := url.Parse(databaseURL) + if err != nil { + return nil, fmt.Errorf("error parsing MySQL database URL: %w", err) + } + config := mysql.NewConfig() + config.Addr = net.JoinHostPort(cmp.Or(parsed.Hostname(), "localhost"), cmp.Or(parsed.Port(), "3306")) + config.DBName = strings.TrimPrefix(parsed.Path, "/") + config.Net = "tcp" + config.ParseTime = true + config.MultiStatements = true + if parsed.User != nil { + config.User = parsed.User.Username() + config.Passwd, _ = parsed.User.Password() + } + // Parse the URL query with the driver's own option handling (including TLS), + // then enforce the timestamp and migration options River requires. + if parsed.RawQuery != "" { + config, err = mysql.ParseDSN(config.FormatDSN() + "&" + parsed.RawQuery) + if err != nil { + return nil, fmt.Errorf("error parsing MySQL connection options: %w", err) + } + } + config.Loc = time.UTC + config.MultiStatements = true + config.ParseTime = true + return config, nil +} + // Determines if there's a minimum number of `PG*` env vars configured to // consider that configurable path viable. A `--database-url` parameter will // take precedence. diff --git a/cmd/river/rivercli/command_test.go b/cmd/river/rivercli/command_test.go new file mode 100644 index 000000000..e5c5867eb --- /dev/null +++ b/cmd/river/rivercli/command_test.go @@ -0,0 +1,43 @@ +package rivercli + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParseMySQLURL(t *testing.T) { + t.Parallel() + + t.Run("ConnectionOptions", func(t *testing.T) { + t.Parallel() + + config, err := parseMySQLURL("mysql://worker:p%40ss%3Fword@[::1]:3307/river%20test?tls=preferred&timeout=5s") + require.NoError(t, err) + require.Equal(t, "[::1]:3307", config.Addr) + require.Equal(t, "river test", config.DBName) + require.Equal(t, "worker", config.User) + require.Equal(t, "p@ss?word", config.Passwd) + require.Equal(t, "preferred", config.TLSConfig) + require.Equal(t, 5*time.Second, config.Timeout) + }) + + t.Run("InvalidConnectionOption", func(t *testing.T) { + t.Parallel() + + _, err := parseMySQLURL("mysql://localhost/river?timeout=invalid") + require.ErrorContains(t, err, "MySQL connection options") + }) + + t.Run("RequiredOptions", func(t *testing.T) { + t.Parallel() + + config, err := parseMySQLURL("mysql://localhost/river?parseTime=false&multiStatements=false&loc=America%2FChicago") + require.NoError(t, err) + require.Equal(t, "localhost:3306", config.Addr) + require.Same(t, time.UTC, config.Loc) + require.True(t, config.MultiStatements) + require.True(t, config.ParseTime) + }) +} diff --git a/cmd/river/rivercli/driver_procurer.go b/cmd/river/rivercli/driver_procurer.go index 181b43170..874480587 100644 --- a/cmd/river/rivercli/driver_procurer.go +++ b/cmd/river/rivercli/driver_procurer.go @@ -9,6 +9,7 @@ import ( "github.com/riverqueue/river/cmd/river/riverbench" "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riversqlite" "github.com/riverqueue/river/rivermigrate" @@ -23,6 +24,11 @@ type DriverProcurer interface { QueryRow(ctx context.Context, sql string, args ...any) riverdriver.Row } +// DriverProcurerMySQL optionally initializes a custom procurer with a MySQL pool. +type DriverProcurerMySQL interface { + InitMySQL(pool *sql.DB) +} + // DriverProcurerPgxV5 optionally initializes a custom procurer with a pgx/v5 pool. type DriverProcurerPgxV5 interface { InitPgxV5(pool *pgxpool.Pool) @@ -51,6 +57,22 @@ type MigratorInterface interface { Validate(ctx context.Context, opts *rivermigrate.ValidateOpts) (*rivermigrate.ValidateResult, error) } +type mysqlDriverProcurer struct { + dbPool *sql.DB +} + +func (p *mysqlDriverProcurer) GetBenchmarker(config *riverbench.Config) BenchmarkerInterface { + return riverbench.NewBenchmarker(rivermysql.New(p.dbPool), config) +} + +func (p *mysqlDriverProcurer) GetMigrator(config *rivermigrate.Config) (MigratorInterface, error) { + return rivermigrate.New(rivermysql.New(p.dbPool), config) +} + +func (p *mysqlDriverProcurer) QueryRow(ctx context.Context, query string, args ...any) riverdriver.Row { + return rivermysql.New(p.dbPool).GetExecutor().QueryRow(ctx, query, args...) +} + type pgxV5DriverProcurer struct { dbPool *pgxpool.Pool } diff --git a/cmd/river/rivercli/river_cli.go b/cmd/river/rivercli/river_cli.go index 224822269..41c47fc74 100644 --- a/cmd/river/rivercli/river_cli.go +++ b/cmd/river/rivercli/river_cli.go @@ -151,7 +151,7 @@ default. } addDatabaseURLFlag := func(cmd *cobra.Command, databaseURL *string) { - cmd.Flags().StringVar(databaseURL, "database-url", "", "URL of the database (should look like `postgres://...`)") + cmd.Flags().StringVar(databaseURL, "database-url", "", "URL of the database (`mysql://...`, `postgres://...`, or `sqlite://...`)") } addLineFlag := func(cmd *cobra.Command, line *string) { cmd.Flags().StringVar(line, "line", "", "migration line to operate on (default: main)") diff --git a/cmd/river/rivercli/river_cli_test.go b/cmd/river/rivercli/river_cli_test.go index f8ce43714..40f9d6ce6 100644 --- a/cmd/river/rivercli/river_cli_test.go +++ b/cmd/river/rivercli/river_cli_test.go @@ -170,7 +170,7 @@ func TestBaseCommandSetIntegration(t *testing.T) { cmd, _ := setup(t) cmd.SetArgs([]string{"migrate-down", "--database-url", "post://"}) - require.EqualError(t, cmd.Execute(), "unsupported database URL (`post://`); try one with a `postgres://`, `postgresql://`, or `sqlite://` scheme/prefix") + require.EqualError(t, cmd.Execute(), "unsupported database URL (`post://`); try one with a `mysql://`, `postgres://`, `postgresql://`, or `sqlite://` scheme/prefix") }) t.Run("MigrateUpWithAppliedTargetVersion", func(t *testing.T) { diff --git a/conformance/go.sum b/conformance/go.sum index 3cd683838..c328dd1d3 100644 --- a/conformance/go.sum +++ b/conformance/go.sum @@ -8,16 +8,16 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= diff --git a/docs/development.md b/docs/development.md index 8b0454959..6049a13ed 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,27 +13,183 @@ Run tests: make test -By default, this runs all workspace modules and database drivers. Use -`make test/race` to enable the race detector. +By default, this runs all workspace modules and database drivers except MySQL, +which is opt-in. Use `make test/race` to enable the race detector. + +To run only the MySQL driver tests (no Postgres needed): + + make test TEST_DATABASE=mysql + +These connect as `root` without a password on `localhost:3306`. Set +`TEST_MYSQL_URL` to override the connection DSN. To include MySQL in the full +suite, set `RIVER_MYSQL_TESTS_ENABLED=1`. To run only the SQLite, libSQL, and Turso driver tests (no Postgres needed): make test TEST_DATABASE=sqlite -To run the regular suite with the SQLite driver tests excluded: +To run the regular suite with the MySQL and SQLite driver tests excluded: make test TEST_DATABASE=postgres `TEST_DATABASE` also applies to `make test/race`. CI separates these suites so -the Postgres version matrix doesn't repeat the SQLite tests. Each suite still -runs on both supported Go versions. Only the latest combination in each matrix -uses the race detector, marked `race` in its job title. +the Postgres version matrix doesn't repeat the MySQL and SQLite tests. Postgres +and SQLite run on both supported Go versions, with the latest combination in +each matrix using the race detector. MySQL runs separately with the race +detector on the latest supported Go version. + +## Compare database performance + +Run the same Go workload against MySQL, Postgres, and file-backed SQLite: + + make bench/compare + +The command needs local Postgres and MySQL servers. It defaults to +`postgres://localhost/postgres` and `root@tcp(127.0.0.1:3306)/`. Override these +with `RIVER_BENCH_POSTGRES_URL` and `RIVER_BENCH_MYSQL_DSN` (the latter uses the +Go MySQL driver's DSN syntax). Credentials are not written into the results. +The Postgres role needs permission to create schemas in the selected database; +the MySQL user needs permission to create databases. + +Each trial creates and migrates its own randomly named Postgres schema, MySQL +database, or temporary SQLite file, and removes it afterward. Existing River +tables are never truncated. SIGINT/SIGTERM also trigger cleanup; an uncatchable +termination may leave a `river_bench_*` schema/database behind. + +By default, the benchmark uses: + +- 30,000 identical, non-unique jobs with a 256-byte payload string per trial. +- `InsertMany` batches of 1,000, including returned job decoding and commit. +- 100 no-op workers, 2 ms fetch cooldown, and 20 ms polling interval. +- `GOMAXPROCS=4`, pools of 10 for Postgres/MySQL, and one connection for SQLite. +- File-backed SQLite with WAL and `synchronous=FULL`. Postgres and MySQL retain + their configured durability settings, which are recorded in the JSON. +- One discarded warm-up round and three measured rounds. Database order rotates + every round, and only one benchmark runs at a time. + +Insertion throughput measures enqueueing all jobs before workers start. Drain +throughput measures client startup through receipt of all committed completion +events. Shutdown, migration, verification, and cleanup are outside these timers. +A trial fails on a job failure, retry, timeout, or incorrect persisted completion +count. These are separate batch-insert and backlog-drain workloads, **not** a +steady-state benchmark of producers inserting while workers run. They measure +River's Go implementations, including their different driver strategies, rather +than database engines in isolation. They do not measure application job work, +uniqueness conflicts, or latency distributions. + +The command prints median and min/max throughput and saves every trial, database +versions/settings, Git status, Go build information, and before/after load +averages to a new JSON file. The default output path is in the system temporary +directory. Existing result files are never overwritten. Save a named result and +repeat the same settings later when other builds and tests are idle: + + make bench/compare BENCH_COMPARE_ARGS="-output /tmp/river-bench-busy.json" + make bench/compare BENCH_COMPARE_ARGS="-output /tmp/river-bench-quiet.json" + +For longer runs or a concurrency sweep, use the command's flags: + + make bench/compare BENCH_COMPARE_ARGS="-jobs 100000 -samples 6 -workers 1000 -output /tmp/river-bench-long.json" + make bench/compare BENCH_COMPARE_ARGS="-h" + +To isolate a driver during optimization, select just that backend: + + make bench/compare BENCH_COMPARE_ARGS="-backends mysql -samples 6 -output /tmp/river-bench-mysql.json" + +To measure the count-only `InsertManyFast` API, which avoids fetching inserted +job rows and IDs, use: + + make bench/compare BENCH_COMPARE_ARGS="-insert-mode fast -output /tmp/river-bench-fast.json" + +The default is `-insert-mode returning` (`InsertMany`). The mode is recorded in +the results. Both modes use the same non-unique jobs and verify their eventual +completion. Compare modes separately: requesting inserted rows does additional +work, particularly on MySQL where the driver must emulate `RETURNING`. + +For MySQL, also compare the Go driver's optional parameter interpolation. This +avoids preparing and closing a server-side statement for each parameterized +query. The benchmark records `interpolate_params` without saving the DSN: + + RIVER_BENCH_MYSQL_DSN='root@tcp(127.0.0.1:3306)/?interpolateParams=true' make bench/compare BENCH_COMPARE_ARGS="-backends mysql -output /tmp/river-bench-mysql-interpolated.json" + +When comparing driver changes, build both versions first and alternate their +execution order. Keep connection settings, including parameter interpolation, +the same for both versions; measure connection tuning separately. + +Keep the same source revision, batch size, concurrency, payload, server settings, +and machine when comparing reruns. Warm-ups do not clear OS or database caches; +these are warm-server measurements on fresh tables. Shared-machine results are +provisional, and load averages alone cannot correct for CPU or disk contention. +The servers' durability mechanisms and cache sizes also differ, so inspect the +recorded settings before interpreting differences as engine speed. + +## MySQL design and operation + +MySQL support uses InnoDB and MySQL 8.0+ features; MariaDB is not a supported +substitute. The canonical migrations are in `riverdriver/rivermysql/migration`. +`make generate` copies them to Java, JavaScript, Ruby, and Rust. Migration +versions match the other backends even when a backend needs a no-op migration. + +The schema deliberately uses these choices: + +- `DATETIME(6)` stores UTC with microsecond precision without `TIMESTAMP`'s + 2038 upper bound or session time zone conversion. Drivers must encode and + decode these values as UTC. +- `utf8mb4_0900_bin` gives identifiers exact, case-sensitive comparisons, + including trailing spaces. The older `utf8mb4_bin` pads comparisons and can + collapse distinct queue names. Every table also declares its character set, + so notification payloads support Unicode regardless of database defaults. +- Native `JSON` validates documents and supports indexed/filterable operations. + MySQL stores integers within its native signed/unsigned 64-bit range exactly, + but can normalize larger numbers to floating point. Applications requiring + arbitrary-precision values should encode them as JSON strings. Metadata + merges are shallow and preserve explicit nulls; `JSON_MERGE_PATCH` would + incorrectly merge nested objects and delete null-valued keys. +- Job states are checked strings. The unique-job index evaluates the unique + state bitmask and yields SQL NULL for jobs outside that set. This preserves + conditional uniqueness without a partial-index feature. The fetching index + starts with state and queue, followed by priority, schedule, and ID to match + the claiming order. Keep these predicates indexable: they determine which + rows locking reads visit. + +River-owned write transactions use READ COMMITTED and row locks. Operations +joining a caller's transaction preserve its isolation and commit/rollback +ownership. Without `RETURNING`, a mutation and the read of its result must +remain in one transaction. Locking reads and `SKIP LOCKED` let workers and +maintenance tasks avoid waiting on jobs another transaction owns. + +Notifications use a transactionally written outbox and an independent cursor +per listener. Auto-increment IDs are allocated before commit, so a higher ID +can commit first. Listeners use locking scans when establishing and advancing +the cursor, waiting for earlier uncommitted rows; a snapshot `MAX(id)` or +`SKIP LOCKED` scan could lose notifications. Cleanup is bounded and skips +locked rows. + +MySQL DDL commits implicitly. Run one migration process at a time; the port +migrators also use named locks, while Go's migrator relies on deployment +coordination. A failed migration can leave partial DDL applied; inspect and +repair the schema before retrying. Application transactions cannot roll back +schema changes. Go rejects MySQL calls to the deprecated `MigrateTx` API before +executing SQL; use `Migrate` on a pool instead. + +For maximum-size job output, configure `max_allowed_packet` to at least +128 MiB on the server (and compatible client limits). Ruby SQL adapters can +escape a 32 MiB output to more than 64 MiB of SQL. For a local test server: + + mysql -uroot -e 'SET GLOBAL max_allowed_packet = 134217728' + +Use a fresh connection after changing this setting. The Ruby MySQL CI job sets +it explicitly. Go tests accept the driver's DSN in `TEST_MYSQL_URL`; the ports +use URLs, with Ruby requiring the `mysql2://` scheme. + +See MySQL's [binary collation rules](https://dev.mysql.com/doc/refman/8.4/en/charset-binary-collations.html), +[native JSON representation](https://dev.mysql.com/doc/refman/8.4/en/json.html), +and [locking reads](https://dev.mysql.com/doc/refman/8.4/en/innodb-locking-reads.html). ## Run lint Run the linter and try to autofix: - golangci-lint run --fix + make lint ## Generate sqlc @@ -73,6 +229,7 @@ flags: git checkout master && git pull --rebase git tag cmd/river/$VERSION -m "release cmd/river/$VERSION" git tag riverdriver/$VERSION -m "release riverdriver/$VERSION" + git tag riverdriver/rivermysql/$VERSION -m "release riverdriver/rivermysql/$VERSION" git tag riverdriver/riverpgxv5/$VERSION -m "release riverdriver/riverpgxv5/$VERSION" git tag riverdriver/riverdatabasesql/$VERSION -m "release riverdriver/riverdatabasesql/$VERSION" git tag riverdriver/riversqlite/$VERSION -m "release riverdriver/riversqlite/$VERSION" diff --git a/go.sum b/go.sum index ae18a83d2..c302c8914 100644 --- a/go.sum +++ b/go.sum @@ -10,14 +10,14 @@ github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QII github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/go.work b/go.work index 7f9949533..905844064 100644 --- a/go.work +++ b/go.work @@ -10,6 +10,7 @@ use ( ./riverdriver/riverdatabasesql ./riverdriver/riverdrivertest ./riverdriver/riverpgxv5 + ./riverdriver/rivermysql ./riverdriver/riversqlite ./rivershared ./rivertype diff --git a/internal/cmd/syncrustmigrations/main.go b/internal/cmd/syncrustmigrations/main.go index 576738863..7c64b531a 100644 --- a/internal/cmd/syncrustmigrations/main.go +++ b/internal/cmd/syncrustmigrations/main.go @@ -22,6 +22,10 @@ func main() { flag.Parse() databases := []database{ + { + canonicalDir: "riverdriver/rivermysql/migration/main", + mirrorDir: "rust/riverqueue-migrate/migrations/mysql/main", + }, { canonicalDir: "riverdriver/riverpgxv5/migration/main", mirrorDir: "rust/riverqueue-migrate/migrations/main", diff --git a/internal/dblist/db_list.go b/internal/dblist/db_list.go index b91596dee..1e1febc81 100644 --- a/internal/dblist/db_list.go +++ b/internal/dblist/db_list.go @@ -44,6 +44,7 @@ type WherePredicate struct { } type sqlFragmentBuilder interface { + DatabaseName() string SQLFragmentColumnContainsAll(column, namedArg string, values []string) (string, any, error) SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) SQLFragmentColumnIn(column string, values any) (string, any, error) @@ -197,6 +198,17 @@ func JobMakeDriverParams(ctx context.Context, params *JobListParams, sqlFragment var orderByBuilder strings.Builder for i, orderBy := range params.OrderBy { + // MySQL doesn't support NULLS FIRST/LAST. Sort by nullness first to + // preserve the same cursor ordering as Postgres and SQLite. + mysqlNullOrder := sqlFragmentBuilder.DatabaseName() == riverdriver.DatabaseNameMySQL && orderBy.Expr != "id" + if mysqlNullOrder { + orderByBuilder.WriteString(orderBy.Expr) + if orderBy.Order == SortOrderAsc { + orderByBuilder.WriteString(" IS NULL ASC, ") + } else { + orderByBuilder.WriteString(" IS NULL DESC, ") + } + } orderByBuilder.WriteString(orderBy.Expr) // Match Postgres's default null placement on every driver. Leave the // non-null ID tie-breaker alone: SQLite otherwise adds a temporary sort @@ -204,12 +216,12 @@ func JobMakeDriverParams(ctx context.Context, params *JobListParams, sqlFragment switch orderBy.Order { case SortOrderAsc: orderByBuilder.WriteString(" ASC") - if orderBy.Expr != "id" { + if orderBy.Expr != "id" && !mysqlNullOrder { orderByBuilder.WriteString(" NULLS LAST") } case SortOrderDesc: orderByBuilder.WriteString(" DESC") - if orderBy.Expr != "id" { + if orderBy.Expr != "id" && !mysqlNullOrder { orderByBuilder.WriteString(" NULLS FIRST") } case SortOrderUnspecified: diff --git a/internal/maintenance/job_rescuer_test.go b/internal/maintenance/job_rescuer_test.go index b5d40f892..1903295b7 100644 --- a/internal/maintenance/job_rescuer_test.go +++ b/internal/maintenance/job_rescuer_test.go @@ -7,6 +7,7 @@ import ( "math" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/require" @@ -77,6 +78,14 @@ func (p *SimpleClientRetryPolicy) NextRetry(job *rivertype.JobRow) time.Time { return job.AttemptedAt.Add(timeutil.SecondsAsDuration(retrySeconds)) } +type jobRescuerExecutorMock struct { + riverdriver.Executor +} + +func (e *jobRescuerExecutorMock) JobGetStuck(context.Context, *riverdriver.JobGetStuckParams) ([]*rivertype.JobRow, error) { + return nil, nil +} + type jobRescuerPilotSpy struct { riverpilot.StandardPilot @@ -359,16 +368,38 @@ func TestJobRescuer(t *testing.T) { t.Run("CustomizableInterval", func(t *testing.T) { t.Parallel() - cleaner, _ := setup(t) - cleaner.Config.Interval = 1 * time.Microsecond - - require.NoError(t, cleaner.Start(ctx)) - - // This should trigger ~immediately every time: - for i := range 5 { - t.Logf("Iteration %d", i) - cleaner.TestSignals.FetchedBatch.WaitOrTimeout() - } + synctest.Test(t, func(t *testing.T) { + // Keep database I/O out of the fake-clock test so each interval runs + // only when time advances, without overflowing either test signal. + const interval = time.Minute + rescuer := NewRescuer(riversharedtest.BaseServiceArchetype(t), &JobRescuerConfig{ + ClientRetryPolicy: &SimpleClientRetryPolicy{}, + Interval: interval, + WorkUnitFactoryFunc: func(string) workunit.WorkUnitFactory { + return nil + }, + }, &jobRescuerExecutorMock{}) + rescuer.StaggerStartupDisable(true) + rescuer.TestSignals.Init(t) + t.Cleanup(rescuer.Stop) + + require.NoError(t, rescuer.Start(ctx)) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.WaitOrTimeout() + rescuer.TestSignals.UpdatedBatch.WaitOrTimeout() + + for range 5 { + time.Sleep(interval - time.Nanosecond) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.RequireEmpty() + rescuer.TestSignals.UpdatedBatch.RequireEmpty() + + time.Sleep(time.Nanosecond) + synctest.Wait() + rescuer.TestSignals.FetchedBatch.WaitOrTimeout() + rescuer.TestSignals.UpdatedBatch.WaitOrTimeout() + } + }) }) t.Run("StopsImmediately", func(t *testing.T) { diff --git a/internal/maintenance/sqlite_notification_cleaner.go b/internal/maintenance/sqlite_notification_cleaner.go index 16e80ece9..88696fbc6 100644 --- a/internal/maintenance/sqlite_notification_cleaner.go +++ b/internal/maintenance/sqlite_notification_cleaner.go @@ -67,9 +67,8 @@ func (c *SQLiteNotificationCleanerConfig) mustValidate() *SQLiteNotificationClea return c } -// SQLiteNotificationCleaner periodically removes old rows from SQLite's -// notification outbox. It is only needed for the SQLite driver's emulated -// listen/notify support. +// SQLiteNotificationCleaner periodically removes old rows from the durable +// notification outbox used to emulate listen/notify for SQLite and MySQL. type SQLiteNotificationCleaner struct { riversharedmaintenance.QueueMaintainerServiceBase startstop.BaseStartStop @@ -84,7 +83,7 @@ type SQLiteNotificationCleaner struct { reducedBatchSizeBreaker *circuitbreaker.CircuitBreaker } -// NewSQLiteNotificationCleaner returns a SQLite notification cleaner. +// NewSQLiteNotificationCleaner returns a notification outbox cleaner. func NewSQLiteNotificationCleaner(archetype *baseservice.Archetype, config *SQLiteNotificationCleanerConfig, exec riverdriver.Executor) *SQLiteNotificationCleaner { batchSizes := config.WithDefaults() @@ -132,7 +131,7 @@ func (s *SQLiteNotificationCleaner) Start(ctx context.Context) error { //nolint: if ctx.Err() != nil { return } - s.Logger.ErrorContext(ctx, s.Name+": Error cleaning SQLite notifications", slog.String("error", err.Error())) + s.Logger.ErrorContext(ctx, s.Name+": Error cleaning notification outbox", slog.String("error", err.Error())) continue } diff --git a/java/DIFFERENCES.md b/java/DIFFERENCES.md index 123daf41c..57063e792 100644 --- a/java/DIFFERENCES.md +++ b/java/DIFFERENCES.md @@ -54,7 +54,7 @@ `attempt` and `attempted_at`. Go updates by job ID alone. Java rejects a stale attempt's checkpoint to prevent it from changing a replacement attempt. - SQLite follows Go's JSONB storage and millisecond timestamp representation; - Postgres uses microsecond timestamps. These are shared storage contracts, + MySQL and Postgres use microsecond timestamps. These are shared storage contracts, not configurable Java serialization choices. Stored uniqueness hashes, job state values, migration versions, and notification diff --git a/java/README.md b/java/README.md index 947bbe6e6..b274ffb9c 100644 --- a/java/README.md +++ b/java/README.md @@ -1,6 +1,6 @@ # River for Java -Java 21 client and worker runtime for River's Postgres and SQLite +Java 21 client and worker runtime for River's MySQL, Postgres, and SQLite schemas. Jobs are ordinary River jobs: another language can insert, cancel, retry, or work them using the same database. @@ -32,14 +32,14 @@ Java 21 or newer is required. Add `com.riverqueue:river:0.4.0` and a JDBC driver The River repository is public and requires no credentials. Maven Central supplies third-party dependencies. -For SQLite, replace the Postgres JDBC dependency with `org.xerial:sqlite-jdbc:3.53.4.0`. To install a local copy of River from source, see the [development guide](docs/development.md#building-from-source). +For MySQL, use `com.mysql:mysql-connector-j:26.7.0`. For SQLite, replace the Postgres JDBC dependency with `org.xerial:sqlite-jdbc:3.53.4.0`. To install a local copy of River from source, see the [development guide](docs/development.md#building-from-source). ## Quick start Define job arguments as a record, give the job a stable kind, and register a worker lambda. This complete `Example.java` starts a client, inserts a job, waits for its completion, and stops the workers. Set `DATABASE_URL` to a -Postgres URL or a file-backed SQLite JDBC URL. +MySQL or Postgres URL, or a file-backed SQLite JDBC URL. ```java import com.riverqueue.*; @@ -135,8 +135,8 @@ river-java migrate-get --version 3 --up > river_3.up.sql river-java migrate-get --driver sqlite --all --up > river_sqlite.up.sql ``` -SQL export defaults to Postgres without a database URL. Use `--driver sqlite` -for SQLite, or select the dialect through the URL. Export accepts comma-separated +SQL export defaults to Postgres without a database URL. Use `--driver mysql` or `--driver sqlite` +for MySQL or SQLite, or select the dialect through the URL. Export accepts comma-separated `--version` values or `--all`; `--exclude-version 1` excludes migration history setup when exporting for another migration framework. Exit codes are `0` for success, `1` for an operation failure, and `2` for invalid arguments. @@ -827,6 +827,26 @@ Workers.Handler sendEmail = context -> clientBuilder.worker(email, sendEmail); ``` +## MySQL + +Use MySQL 8.0 or later with the latest patch release. `Database.connect` accepts +`mysql://user:password@localhost/my_app` and `jdbc:mysql://localhost/my_app`. +Applications with an existing pool can use `new Database(dataSource, Database.Dialect.MYSQL)`. +`withSchema("background_jobs")` selects a separate MySQL database for River's tables. + +Workers claim jobs with `FOR UPDATE SKIP LOCKED` and receive notifications +through the durable outbox, including cancellation and leadership resignation. +River-owned transactions use READ COMMITTED; caller transactions keep their +configured isolation level. Inserts and job completion participate in caller +transactions through the same APIs as Postgres. + +MySQL DDL commits implicitly. Migration history is recorded after each successful +version, but a failed version may leave partially applied DDL. Inspect and repair +it before retrying. A named database lock prevents concurrent migration runners. + +See the shared [MySQL design notes](../docs/development.md#mysql-design-and-operation) for JSON numeric limits, +exact string comparisons, and server packet-size requirements. + ## SQLite Use a [file database](https://riverqueue.com/docs/sqlite) shared by every diff --git a/java/bin/check-packages/main.go b/java/bin/check-packages/main.go index 0c6ed6f73..49106cdf6 100644 --- a/java/bin/check-packages/main.go +++ b/java/bin/check-packages/main.go @@ -15,7 +15,10 @@ import ( "strings" ) -var forbiddenPath = regexp.MustCompile(`(^|/)(fixtures?|testdata|tests|conformance)/|(?:Test|Tests)(?:\$[^/]*)?\.(class|java)$`) +// Dependencies in the shaded CLI may have production classes ending in Test +// (for example, protobuf's UnredactedDebugFormatForTest). Check that suffix +// only in River's namespace. +var forbiddenPath = regexp.MustCompile(`(^|/)(fixtures?|testdata|tests|conformance)/|^com/riverqueue/.*(?:Test|Tests)(?:\$[^/]*)?\.(class|java)$`) func main() { if err := run("java"); err != nil { diff --git a/java/bin/check-packages/main_test.go b/java/bin/check-packages/main_test.go index 65b76b96e..87ed8bcda 100644 --- a/java/bin/check-packages/main_test.go +++ b/java/bin/check-packages/main_test.go @@ -20,6 +20,7 @@ func TestCheckArchive(t *testing.T) { wantError string }{ {name: "Conformance", extra: "com/riverqueue/conformance/Adapter.class", wantError: "development content"}, + {name: "DependencyProductionClass", extra: "com/google/protobuf/UnredactedDebugFormatForTest.class", resource: "SELECT 1;"}, {name: "Fixture", extra: "fixtures/golden.json", wantError: "development content"}, {name: "InnerTestClass", extra: "com/riverqueue/ClientTest$Helper.class", wantError: "development content"}, {name: "JSON", extra: "com/riverqueue/protocol.json", wantError: "development content"}, diff --git a/java/bin/github-release.py b/java/bin/github-release.py index 3364e5108..637f59faa 100644 --- a/java/bin/github-release.py +++ b/java/bin/github-release.py @@ -156,7 +156,7 @@ def prepare(directory): "verify", "org.apache.maven.plugins:maven-deploy-plugin:3.2.0:deploy", "-Driver.test.database=sqlite", - "-DexcludedGroups=postgres", + "-DexcludedGroups=mysql,postgres", "-DaltDeploymentRepository=" + destination, "-DaltReleaseDeploymentRepository=" + destination, ], diff --git a/java/bin/sync-migrations/main.go b/java/bin/sync-migrations/main.go index 6574130e9..c88d7c01f 100644 --- a/java/bin/sync-migrations/main.go +++ b/java/bin/sync-migrations/main.go @@ -38,6 +38,7 @@ func run(root string, check bool) error { dialect string driver string }{ + {"mysql", "rivermysql"}, {"postgres", "riverpgxv5"}, {"sqlite", "riversqlite"}, } { @@ -68,7 +69,7 @@ func run(root string, check bool) error { } } if names != nil && !slices.Equal(current, names) { - return errors.New("migration names differ between Postgres and SQLite; review the Java catalog") + return errors.New("migration names differ between database backends; review the Java catalog") } names = current } diff --git a/java/bin/sync-migrations/main_test.go b/java/bin/sync-migrations/main_test.go index 6db5a1d7c..867c3f683 100644 --- a/java/bin/sync-migrations/main_test.go +++ b/java/bin/sync-migrations/main_test.go @@ -24,7 +24,7 @@ func TestRun(t *testing.T) { destination: filepath.Join(root, "java/river/src/main/resources/com/riverqueue/migration"), root: root, } - for _, driver := range []string{"riverpgxv5", "riversqlite"} { + for _, driver := range []string{"rivermysql", "riverpgxv5", "riversqlite"} { source := filepath.Join(root, "riverdriver", driver, "migration/main") require.NoError(t, os.MkdirAll(source, 0o755)) for _, direction := range []string{"up", "down"} { @@ -50,7 +50,7 @@ func TestRun(t *testing.T) { missing := filepath.Join(bundle.destination, "postgres/001_first.down.sql") stale := filepath.Join(bundle.destination, "sqlite/001_first.up.sql") - extra := filepath.Join(bundle.destination, "sqlite/999_extra.up.sql") + extra := filepath.Join(bundle.destination, "mysql/999_extra.up.sql") require.NoError(t, os.Remove(missing)) require.NoError(t, os.WriteFile(stale, []byte("stale"), 0o600)) require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600)) diff --git a/java/cli/pom.xml b/java/cli/pom.xml index 0583ebb4b..cf21eab13 100644 --- a/java/cli/pom.xml +++ b/java/cli/pom.xml @@ -6,6 +6,7 @@ Executable database migration tool for River on Postgres and SQLite com.riverqueueriver0.4.0 + com.mysqlmysql-connector-j org.postgresqlpostgresql org.xerialsqlite-jdbc org.junit.jupiterjunit-jupitertest diff --git a/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java b/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java index bdee00485..ffdf18a53 100644 --- a/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java +++ b/java/cli/src/main/java/com/riverqueue/cli/MigrationCli.java @@ -33,19 +33,26 @@ private static Database database( Options options, Map environment, boolean offline) { String url = options.values.getOrDefault("database-url", environment.get("DATABASE_URL")); String driver = options.values.get("driver"); - if (driver != null && !Set.of("postgres", "sqlite").contains(driver)) - throw new Usage("--driver must be postgres or sqlite"); + if (driver != null && !Set.of("mysql", "postgres", "sqlite").contains(driver)) + throw new Usage("--driver must be mysql, postgres, or sqlite"); if (offline && driver != null && !options.values.containsKey("database-url")) url = null; if (url == null || url.isBlank()) { if (!offline) throw new Usage("Set DATABASE_URL or pass --database-url"); - url = "sqlite".equals(driver) ? "jdbc:sqlite::memory:" : "jdbc:postgresql://localhost/unused"; + url = + "mysql".equals(driver) + ? "jdbc:mysql://localhost/unused" + : "sqlite".equals(driver) + ? "jdbc:sqlite::memory:" + : "jdbc:postgresql://localhost/unused"; } - if (!url.startsWith("postgres://") + if (!url.startsWith("mysql://") + && !url.startsWith("jdbc:mysql:") + && !url.startsWith("postgres://") && !url.startsWith("postgresql://") && !url.startsWith("jdbc:postgresql:") && !url.startsWith("sqlite:") && !url.startsWith("jdbc:sqlite:")) - throw new Usage("Expected a Postgres URL or a SQLite JDBC URL"); + throw new Usage("Expected a MySQL, Postgres, or SQLite URL"); Database database; try { database = @@ -55,7 +62,7 @@ private static Database database( throw new Usage("Invalid database URL or schema"); } if (driver != null - && !driver.equals(database.dialect() == Database.Dialect.SQLITE ? "sqlite" : "postgres")) + && !driver.equals(database.dialect().name().toLowerCase(java.util.Locale.ROOT))) throw new Usage("--driver does not match the database URL"); return database; } @@ -101,10 +108,11 @@ private static void help(PrintWriter out, String name, List lines) { out.println(" version Print CLI version"); out.println(); out.println("Connection options:"); - out.println(" --database-url URL Postgres or SQLite JDBC URL; defaults to DATABASE_URL"); + out.println(" --database-url URL MySQL, Postgres, or SQLite URL; defaults to DATABASE_URL"); out.println( - " --driver DRIVER postgres or sqlite; selects dialect for offline SQL export"); - out.println(" --schema NAME Postgres schema (default: connection's current schema)"); + " --driver DRIVER mysql, postgres, or sqlite; selects dialect for offline SQL export"); + out.println( + " --schema NAME MySQL database or Postgres schema (default: connection's current schema)"); out.printf( " --line NAME Migration line: %s (default: main)%n", String.join(", ", lines)); out.println(); diff --git a/java/conformance/pom.xml b/java/conformance/pom.xml index 2fc361aba..2784ee2b2 100644 --- a/java/conformance/pom.xml +++ b/java/conformance/pom.xml @@ -12,6 +12,7 @@ com.zaxxerHikariCP7.1.0 com.riverqueueriver${project.version} + com.mysqlmysql-connector-j org.postgresqlpostgresql org.xerialsqlite-jdbc diff --git a/java/docs/development.md b/java/docs/development.md index ea72de746..539d5f87d 100644 --- a/java/docs/development.md +++ b/java/docs/development.md @@ -252,8 +252,9 @@ fresh schema for each test that is removed afterward. Otherwise they use SQLite. and excludes Postgres-only tests, even if a URL is set in your environment. `test/java/conformance` needs no external database: it runs the JUnit tests tagged `conformance`, using temporary SQLite databases for storage checks even if a -Postgres URL is set. The full SQLite and Postgres targets run these checks -against their selected backend. These targets first generate fresh fixtures +database URL is set. `test/java/conformance/mysql` runs the same tagged tests on +MySQL using `RIVER_TEST_DATABASE_URL`. The full SQLite, Postgres, and MySQL targets +run these checks against their selected backend. These targets first generate fresh fixtures directly from this checkout's Go implementation, just like `test/js/conformance` and `test/rust/conformance`. @@ -295,7 +296,7 @@ matrix running client, worker, and migration tests on JDK 25. All jobs use the shared Java setup action and Maven dependency caching. The whole workflow is filtered to changes in Java, its build configuration, fixtures, and canonical migrations. Go changes run the smaller fixture suite in the shared Conformance -workflow alongside Rust and JavaScript. +workflow on SQLite and MySQL alongside Ruby, Rust, and JavaScript. ## Warning policy @@ -361,3 +362,10 @@ does not overwrite the current migrations or generated fixtures. See [validation results](../VALIDATION.md) for the tested revisions and remaining limitations. + +## MySQL tests + +Set `RIVER_TEST_DATABASE_URL=mysql://root@localhost/` and run +`make test/java/mysql` from the repository root. Each test creates and drops its +own database, so the test user needs permission to do both. CI exercises MySQL +8.0 and 8.4 with the shared client, worker, and migration suites. diff --git a/java/pom.xml b/java/pom.xml index 691945188..672dbb1eb 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -27,6 +27,7 @@ tools.jackson.corejackson-databind3.2.4 + com.mysqlmysql-connector-j26.7.0 org.postgresqlpostgresql42.7.14 org.xerialsqlite-jdbc3.53.4.0 org.junit.jupiterjunit-jupiter6.1.3 diff --git a/java/river/pom.xml b/java/river/pom.xml index acb5ce34d..666d08cb0 100644 --- a/java/river/pom.xml +++ b/java/river/pom.xml @@ -6,6 +6,7 @@ Transactional jobs interoperable with River clients in other languages tools.jackson.corejackson-databind + com.mysqlmysql-connector-jtrue org.postgresqlpostgresqltrue org.xerialsqlite-jdbctrue org.junit.jupiterjunit-jupitertest diff --git a/java/river/src/main/java/com/riverqueue/Client.java b/java/river/src/main/java/com/riverqueue/Client.java index 159fcacbe..b0a2531fe 100644 --- a/java/river/src/main/java/com/riverqueue/Client.java +++ b/java/river/src/main/java/com/riverqueue/Client.java @@ -851,7 +851,9 @@ private Job.InsertResult insertOne( : original.uniqueStates().stream().mapToInt(Job.State::bit).reduce(0, (a, b) -> a | b); boolean postgres = database.dialect() == Database.Dialect.POSTGRES; try { - if (!postgres && key != null && (stateMask & Job.State.of(state).bit()) != 0) { + if (database.dialect() == Database.Dialect.SQLITE + && key != null + && (stateMask & Job.State.of(state).bit()) != 0) { try (var statement = Sql.prepare(connection, Sql.query(database, "writer_lock"))) { statement.executeUpdate(); } @@ -893,13 +895,7 @@ private Job.InsertResult insertOne( key == null ? null : HexFormat.of().parseHex(key), mask }; - String insertSql = Sql.query(database, "insert"); - if (postgres) - insertSql = - insertSql.replace( - "{duplicate}", - Sql.query(database, nonce ? "insert_duplicate_nonce" : "insert_duplicate_xmax")); - try (var statement = Sql.prepare(connection, insertSql, params); + try (var statement = Sql.mutation(database, connection, "insert", params); var rows = statement.executeQuery()) { if (!rows.next()) throw new RiverException(RiverException.Code.REJECTED, "Insertion returned no row"); @@ -910,6 +906,15 @@ private Job.InsertResult insertOne( .path(Protocol.METADATA_UNIQUE_NONCE) .equals(metadata.path(Protocol.METADATA_UNIQUE_NONCE)) : rows.getBoolean("duplicate"); + if (duplicate + && database.dialect() == Database.Dialect.MYSQL + && (key == null + || (stateMask & Job.State.of(state).bit()) == 0 + || !key.equals(job.uniqueKey()) + || job.uniqueStates() == null + || !job.uniqueStates().contains(job.state()))) + throw new RiverException( + RiverException.Code.REJECTED, "Insertion violated an unrelated unique constraint"); if (!duplicate) pilot.afterInsert(connection, driver(), job); if (!duplicate && job.state() == Job.State.AVAILABLE) notifyInsert(connection, job.queue()); return new Job.InsertResult(job, duplicate); @@ -972,6 +977,9 @@ private void reset() { /** Runs a callback in a new transaction, committing on success and rolling back on failure. */ public T transaction(Transaction action) { try (var connection = database.connection()) { + int isolation = connection.getTransactionIsolation(); + if (database.dialect() == Database.Dialect.MYSQL) + connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); connection.setAutoCommit(false); var previous = ownedCommit.get(); var commit = new Commit(connection); @@ -993,6 +1001,8 @@ public T transaction(Transaction action) { } finally { if (previous == null) ownedCommit.remove(); else ownedCommit.set(previous); + if (database.dialect() == Database.Dialect.MYSQL) + connection.setTransactionIsolation(isolation); } } catch (SQLException e) { throw databaseError("River transaction", e); diff --git a/java/river/src/main/java/com/riverqueue/Database.java b/java/river/src/main/java/com/riverqueue/Database.java index a7fc047bd..25c2f06a1 100644 --- a/java/river/src/main/java/com/riverqueue/Database.java +++ b/java/river/src/main/java/com/riverqueue/Database.java @@ -29,7 +29,9 @@ private Database(Connector connector, Dialect dialect, String schema) { this.connector = connector; this.dialect = dialect; this.schema = schema; - if (!schema.isEmpty() && !schema.matches("[a-zA-Z_][a-zA-Z0-9_]{0,45}")) + String schemaPattern = + dialect == Dialect.MYSQL ? "[a-zA-Z_][a-zA-Z0-9_]{0,63}" : "[a-zA-Z_][a-zA-Z0-9_]{0,45}"; + if (!schema.isEmpty() && !schema.matches(schemaPattern)) throw new IllegalArgumentException("Invalid schema name"); if (dialect == Dialect.SQLITE && !schema.isEmpty()) throw new IllegalArgumentException("SQLite does not support custom schemas"); @@ -44,6 +46,7 @@ public static Database connect(String url, String applicationName) { String jdbc = url.startsWith("jdbc:") ? url : "jdbc:" + url; return new Database(() -> DriverManager.getConnection(jdbc), Dialect.SQLITE, ""); } + boolean mysql = url.startsWith("mysql:") || url.startsWith("jdbc:mysql:"); String jdbc; var properties = new Properties(); properties.setProperty("ApplicationName", applicationName); @@ -59,7 +62,7 @@ public static Database connect(String url, String applicationName) { properties.setProperty("password", URLDecoder.decode(auth[1], StandardCharsets.UTF_8)); } jdbc = - "jdbc:postgresql://" + (mysql ? "jdbc:mysql://" : "jdbc:postgresql://") + uri.getHost() + (uri.getPort() < 0 ? "" : ":" + uri.getPort()) + uri.getRawPath() @@ -67,7 +70,9 @@ public static Database connect(String url, String applicationName) { } String target = jdbc; return new Database( - () -> DriverManager.getConnection(target, properties), Dialect.POSTGRES, ""); + () -> DriverManager.getConnection(target, properties), + mysql ? Dialect.MYSQL : Dialect.POSTGRES, + ""); } public Connection connection() throws SQLException { @@ -117,11 +122,11 @@ public String schema() { } boolean supportsNotifications(Connection connection) throws SQLException { - return dialect == Dialect.SQLITE || capabilities(connection).notifications(); + return dialect != Dialect.POSTGRES || capabilities(connection).notifications(); } boolean usesUniqueNonce(Connection connection) throws SQLException { - return dialect == Dialect.SQLITE || capabilities(connection).uniqueNonce(); + return dialect != Dialect.POSTGRES || capabilities(connection).uniqueNonce(); } private Capabilities capabilities(Connection connection) throws SQLException { @@ -143,6 +148,7 @@ public Database withSchema(String name) { } String prefix() { + if (dialect == Dialect.MYSQL) return schema.isEmpty() ? "" : "`" + schema + "`."; return schema.isEmpty() ? "" : '"' + schema + "\"."; } @@ -150,6 +156,10 @@ Object timestamp(Instant value) { if (value == null) return null; if (dialect == Dialect.POSTGRES) return OffsetDateTime.ofInstant(value.truncatedTo(ChronoUnit.MICROS), ZoneOffset.UTC); + if (dialect == Dialect.MYSQL) + return DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS") + .withZone(ZoneOffset.UTC) + .format(value); value = value.plusNanos(500000).truncatedTo(ChronoUnit.MILLIS); return DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSS") .withZone(ZoneOffset.UTC) @@ -174,6 +184,7 @@ private interface Connector { } public enum Dialect { + MYSQL, POSTGRES, SQLITE } diff --git a/java/river/src/main/java/com/riverqueue/JobQuery.java b/java/river/src/main/java/com/riverqueue/JobQuery.java index e5115e519..68882d81d 100644 --- a/java/river/src/main/java/com/riverqueue/JobQuery.java +++ b/java/river/src/main/java/com/riverqueue/JobQuery.java @@ -216,7 +216,7 @@ private boolean hasMetadataFilter() { return !metadata.isObject() || !metadata.isEmpty(); } - private static String sqliteMetadata( + private static String jsonMetadata( Database database, List parameters, JsonNode wanted, @@ -225,22 +225,28 @@ private static String sqliteMetadata( int depth) { // Array containment is unordered, but each wanted object must match one complete element. // Correlated predicates retain that boundary instead of comparing independent JSON paths. + // MySQL uses scalar subqueries to avoid semijoin rewrites losing nested correlations. String query; if (wanted.isContainer()) { var children = new ArrayList(); String alias = "metadata_" + depth; if (wanted.isObject()) { for (var entry : wanted.properties()) { - parameters.add(entry.getKey()); + parameters.add( + database.dialect() == Database.Dialect.MYSQL + ? "$." + Json.encode(entry.getKey()) + : entry.getKey()); children.add( Sql.query(database, "filter_metadata_member") .replace( "{predicate}", - sqliteMetadata( + jsonMetadata( database, parameters, entry.getValue(), - alias + ".type", + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(COALESCE(JSON_TYPE(" + alias + ".value), 'NULL'))" + : alias + ".type", alias + ".value", depth + 1))); } @@ -250,11 +256,13 @@ private static String sqliteMetadata( Sql.query(database, "filter_metadata_element") .replace( "{predicate}", - sqliteMetadata( + jsonMetadata( database, parameters, element, - alias + ".type", + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(COALESCE(JSON_TYPE(" + alias + ".value), 'NULL'))" + : alias + ".type", alias + ".value", depth + 1))); } @@ -321,6 +329,17 @@ String sql(Database database, List parameters, boolean forDelete) { : field + " " + sort + (descending ? " NULLS FIRST, " : " NULLS LAST, ")) + "id " + sort; + if (database.dialect() == Database.Dialect.MYSQL && !field.isEmpty()) + orderSQL = + field + + " IS NULL " + + (descending ? "DESC" : "ASC") + + ", " + + field + + " " + + sort + + ", id " + + sort; parameters.add(limit); return Sql.query(database, forDelete ? "list_deletable" : "list") .replace("{where}", where) @@ -337,10 +356,19 @@ private String where(Database database, List parameters) { filter(database, filters, parameters, "tags_all", tagsAll); filter(database, filters, parameters, "tags_any", tagsAny); if (hasMetadataFilter()) { - if (database.dialect() == Database.Dialect.SQLITE) - filters.add( - sqliteMetadata(database, parameters, metadata, "json_type(metadata)", "metadata", 0)); - else { + if (database.dialect() != Database.Dialect.POSTGRES) { + String predicate = + jsonMetadata( + database, + parameters, + metadata, + database.dialect() == Database.Dialect.MYSQL + ? "LOWER(JSON_TYPE(metadata))" + : "json_type(metadata)", + "metadata", + 0); + filters.add(predicate); + } else { filters.add(Sql.query(database, "filter_metadata")); parameters.add(Json.encode(metadata)); } diff --git a/java/river/src/main/java/com/riverqueue/Jobs.java b/java/river/src/main/java/com/riverqueue/Jobs.java index 506af84ff..1eb26e9c4 100644 --- a/java/river/src/main/java/com/riverqueue/Jobs.java +++ b/java/river/src/main/java/com/riverqueue/Jobs.java @@ -35,9 +35,10 @@ public Job cancel(Connection connection, long id) { var before = locked(c, id); if (before.state().isFinalized()) return before; try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "cancel"), + "cancel", river.database().timestamp(clock.instant()), Json.encode(clock.instant()), id); @@ -66,9 +67,10 @@ public Job complete(Connection connection, Job job, Object metadata) { connection, c -> { try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "complete"), + "complete", "completed", river.database().timestamp(clock.instant()), null, @@ -105,7 +107,7 @@ public Job delete(Connection connection, long id) { var job = locked(c, id); if (job.state() == Job.State.RUNNING) throw new RiverException(RiverException.Code.REJECTED, "Cannot delete a running job"); - try (var statement = Sql.prepare(c, Sql.query(river.database(), "delete"), id); + try (var statement = Sql.mutation(river.database(), c, "delete", id); var rows = statement.executeQuery()) { rows.next(); var deleted = river.read(rows); @@ -144,6 +146,13 @@ public List> deleteMany(Connection connection, JobQuery query, boo try (var statement = Sql.prepare(c, sql, params.toArray())) { for (var job : jobs) { statement.setLong(params.size() + 1, job.id()); + if (river.database().dialect() == Database.Dialect.MYSQL) { + if (statement.executeUpdate() != 0) { + river.pilot().afterDelete(c, river.driver(), job); + result.add(job); + } + continue; + } try (var rows = statement.executeQuery()) { // Recheck every filter atomically: a peer may have updated this job since listing. if (!rows.next()) continue; @@ -245,9 +254,10 @@ public Job retry(Connection connection, long id) { c -> { var before = locked(c, id); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "retry"), + "retry", river.database().timestamp(clock.instant()), id, river.database().timestamp(clock.instant())); @@ -283,7 +293,7 @@ public Job update(Connection connection, long id, JobUpdateOptions opt connection, c -> { try (var statement = - Sql.prepare(c, Sql.query(river.database(), "update"), Json.encode(updates), id); + Sql.mutation(river.database(), c, "update", Json.encode(updates), id); var rows = statement.executeQuery()) { if (!rows.next()) throw new RiverException(RiverException.Code.NOT_FOUND, "Job " + id + " not found"); diff --git a/java/river/src/main/java/com/riverqueue/Migrator.java b/java/river/src/main/java/com/riverqueue/Migrator.java index 07ac2caaf..6a8190919 100644 --- a/java/river/src/main/java/com/riverqueue/Migrator.java +++ b/java/river/src/main/java/com/riverqueue/Migrator.java @@ -36,14 +36,10 @@ public Migrator(Database database, String line, List migrations) { private void apply(Connection connection, Migration migration, boolean down) throws SQLException { int version = migration.version(); boolean main = line.equals("main"); - if (down && (!main || version > 1)) { - String query = main && version < 5 ? "migration_delete_legacy" : "migration_line_delete"; - Object[] params = main && version < 5 ? new Object[] {version} : new Object[] {line, version}; - try (var statement = Sql.prepare(connection, Sql.query(database, query), params)) { - statement.executeUpdate(); - } - } + boolean mysql = database.dialect() == Database.Dialect.MYSQL; + if (down && !mysql) removeVersion(connection, version); execute(connection, sql(version, down ? Direction.DOWN : Direction.UP)); + if (down && mysql) removeVersion(connection, version); if (!down) { String query = main && version < 5 ? "migration_insert_legacy" : "migration_line_insert"; Object[] params = main && version < 5 ? new Object[] {version} : new Object[] {line, version}; @@ -53,6 +49,18 @@ private void apply(Connection connection, Migration migration, boolean down) thr } } + private void removeVersion(Connection connection, int version) throws SQLException { + boolean main = line.equals("main"); + if (main && version == 1) return; + boolean legacy = + main && (version < 5 || database.dialect() == Database.Dialect.MYSQL && version == 5); + String query = legacy ? "migration_delete_legacy" : "migration_line_delete"; + Object[] params = legacy ? new Object[] {version} : new Object[] {line, version}; + try (var statement = Sql.prepare(connection, Sql.query(database, query), params)) { + statement.executeUpdate(); + } + } + private boolean booleanQuery(Connection connection, String name) throws SQLException { try (var statement = Sql.prepare(connection, Sql.query(database, name)); var rows = statement.executeQuery()) { @@ -65,7 +73,9 @@ private void execute(Connection connection, String sql) throws SQLException { try (var statement = connection.createStatement()) { // Xerial's executeUpdate executes the entire script, including trigger bodies. if (database.dialect() == Database.Dialect.SQLITE) statement.executeUpdate(sql); - else statement.execute(sql); + else if (database.dialect() == Database.Dialect.MYSQL) { + for (String part : MySql.statements(sql)) statement.execute(part); + } else statement.execute(sql); } } @@ -109,8 +119,7 @@ public List list() { } private static List mainMigrations(Database.Dialect dialect) { - String directory = - "migration/" + (dialect == Database.Dialect.POSTGRES ? "postgres/" : "sqlite/"); + String directory = "migration/" + dialect.name().toLowerCase(java.util.Locale.ROOT) + "/"; var migrations = new ArrayList(); for (int i = 0; i < NAMES.size(); i++) { String name = NAMES.get(i); @@ -168,6 +177,10 @@ public Result migrate(Direction direction, Options options) { if (database.dialect() == Database.Dialect.POSTGRES) { execute(connection, Sql.query(database, "migration_lock")); locked = true; + } else if (database.dialect() == Database.Dialect.MYSQL) { + if (!booleanQuery(connection, "migration_lock")) + throw new SQLException("Another migrator holds the database lock"); + locked = true; } var existing = existing(connection); var planned = plan(existing, down, target, maxSteps); diff --git a/java/river/src/main/java/com/riverqueue/MySql.java b/java/river/src/main/java/com/riverqueue/MySql.java new file mode 100644 index 000000000..d5dd7440c --- /dev/null +++ b/java/river/src/main/java/com/riverqueue/MySql.java @@ -0,0 +1,149 @@ +package com.riverqueue; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; +import javax.sql.rowset.RowSetProvider; +import tools.jackson.databind.node.ObjectNode; + +/** MySQL's locking writes and readback; all SQL remains in the query catalog. */ +final class MySql { + private MySql() {} + + static int clean(Database database, Connection connection, Object... horizons) + throws SQLException { + var ids = new ArrayList(); + try (var selected = rows(database, connection, "clean_jobs_select", horizons)) { + while (selected.next()) ids.add(selected.getLong("id")); + } + if (ids.isEmpty()) return 0; + // Lock candidates first, then use primary-key lookups for the delete. A joined + // delete needs a default database, while a subquery delete can lock skipped rows. + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + try (var statement = + Sql.prepare( + connection, + Sql.query(database, "clean_jobs").replace("{ids}", placeholders), + ids.toArray())) { + return statement.executeUpdate(); + } + } + + static ResultSet mutate(Database database, Connection connection, String name, Object[] params) + throws SQLException { + if (connection.getAutoCommit()) + throw new SQLException("MySQL row-returning writes require a transaction"); + return switch (name) { + case "claim", "claim_known" -> claim(database, connection, name, params); + case "delete" -> delete(database, connection, params); + case "insert" -> { + update(database, connection, name, params); + yield rows(database, connection, "inserted"); + } + case "leader_elect" -> { + update(database, connection, name, params); + yield rows(database, connection, "leader_elected", params[0], params[1]); + } + case "complete", "rescue_update", "update" -> { + int idIndex = name.equals("complete") ? 11 : name.equals("update") ? 1 : 7; + int metadataIndex = name.equals("update") ? 0 : 4; + Object[] merged = params.clone(); + try (var before = rows(database, connection, "lock_get", params[idIndex])) { + if (before.next()) { + ObjectNode metadata = (ObjectNode) Json.parse(before.getString("metadata")); + if (name.equals("complete") + && params[2] != null + && !params[2].equals(metadata.path(Protocol.METADATA_RESUMABLE_STEP).asString())) + metadata.setAll((ObjectNode) Json.parse((String) params[3])); + metadata.setAll((ObjectNode) Json.parse((String) params[metadataIndex])); + merged[metadataIndex] = Json.encode(metadata); + } + } + boolean changed = update(database, connection, name, merged) != 0; + yield rows(database, connection, changed ? "lock_get" : "empty_jobs", params[idIndex]); + } + case "cancel", "checkpoint", "retry", "queue_update", "schedule_available" -> { + Object id = + params[name.equals("retry") ? 1 : name.equals("checkpoint") ? 2 : params.length - 1]; + boolean changed = update(database, connection, name, params) != 0; + String read = + switch (name) { + case "queue_update" -> changed ? "queue_get" : "empty_queues"; + case "schedule_available" -> changed ? "scheduled_queue" : "empty_queues"; + default -> changed ? "lock_get" : "empty_jobs"; + }; + yield rows(database, connection, read, id); + } + default -> throw new IllegalArgumentException("Unknown MySQL mutation: " + name); + }; + } + + private static ResultSet claim( + Database database, Connection connection, String name, Object[] params) throws SQLException { + var ids = new ArrayList(); + // Keep an indexable queue equality alongside the exact binary comparison so + // concurrent fetches do not briefly lock and skip jobs in unrelated queues. + var selection = new ArrayList<>(Arrays.asList(Arrays.copyOfRange(params, 2, params.length))); + selection.addFirst(params[2]); + try (var selected = rows(database, connection, name + "_select", selection.toArray())) { + while (selected.next()) ids.add(selected.getLong("id")); + } + if (ids.isEmpty()) return rows(database, connection, "empty_jobs", 0); + for (long id : ids) update(database, connection, "claim_one", params[0], params[1], id); + return rows(database, connection, "claimed", Json.encode(ids)); + } + + private static ResultSet delete(Database database, Connection connection, Object[] params) + throws SQLException { + var snapshot = RowSetProvider.newFactory().createCachedRowSet(); + try (var selected = rows(database, connection, "delete_rows", params)) { + snapshot.populate(selected); + } + try { + update(database, connection, "delete", params); + return snapshot; + } catch (SQLException error) { + snapshot.close(); + throw error; + } + } + + private static ResultSet rows( + Database database, Connection connection, String name, Object... params) throws SQLException { + return Sql.queryRows(connection, Sql.query(database, name), params); + } + + private static int update(Database database, Connection connection, String name, Object... params) + throws SQLException { + try (var statement = Sql.prepare(connection, Sql.query(database, name), params)) { + return statement.executeUpdate(); + } + } + + /** Splits migration scripts without enabling multi-statement application queries. */ + static List statements(String script) { + var tokens = + Pattern.compile( + "'(?:''|\\\\.|[^'\\\\])*'|\"(?:\"\"|\\\\.|[^\"\\\\])*\"|`(?:``|[^`])*`|--[^\\n]*|#[^\\n]*|/\\*.*?\\*/|;|.", + Pattern.DOTALL) + .matcher(script); + var statements = new ArrayList(); + var buffer = new StringBuilder(); + while (tokens.find()) { + String token = tokens.group(); + if (token.startsWith("--") || token.startsWith("#") || token.startsWith("/*")) { + buffer.append(' '); + } else if (token.equals(";")) { + if (!buffer.toString().isBlank()) statements.add(buffer.toString().strip()); + buffer.setLength(0); + } else buffer.append(token); + } + if (!buffer.toString().isBlank()) statements.add(buffer.toString().strip()); + return List.copyOf(statements); + } +} diff --git a/java/river/src/main/java/com/riverqueue/Queues.java b/java/river/src/main/java/com/riverqueue/Queues.java index d8fbc4dc7..5da578379 100644 --- a/java/river/src/main/java/com/riverqueue/Queues.java +++ b/java/river/src/main/java/com/riverqueue/Queues.java @@ -114,9 +114,10 @@ private Queue updateWithinTransaction(Connection connection, String name, Object var value = Json.tree(metadata); if (!value.isObject()) throw new IllegalArgumentException("Queue metadata must be an object"); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), connection, - Sql.query(river.database(), "queue_update"), + "queue_update", Json.encode(metadata), river.database().timestamp(Instant.now()), name); diff --git a/java/river/src/main/java/com/riverqueue/RunHandle.java b/java/river/src/main/java/com/riverqueue/RunHandle.java index 832631639..a5fe6c568 100644 --- a/java/river/src/main/java/com/riverqueue/RunHandle.java +++ b/java/river/src/main/java/com/riverqueue/RunHandle.java @@ -386,11 +386,10 @@ private void claim(String queue, int count) { ignored -> { var result = new ArrayList(); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query( - river.database(), - config.fetchOnlyKnownKinds ? "claim_known" : "claim"), + config.fetchOnlyKnownKinds ? "claim_known" : "claim", config.fetchOnlyKnownKinds ? new Object[] { river.database().timestamp(Instant.now()), @@ -727,8 +726,7 @@ private void completeBatches() { var rows = new ArrayList>(); for (var completion : batch) try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "complete"), completion.params); + Sql.mutation(river.database(), c, "complete", completion.params); var result = statement.executeQuery()) { if (result.next()) rows.add(river.readPartial(result).job()); else { @@ -786,8 +784,8 @@ private void heartbeat(String queue) { } private void listen() { - if (river.database().dialect() == Database.Dialect.SQLITE) { - listenSqlite(); + if (river.database().dialect() != Database.Dialect.POSTGRES) { + listenOutbox(); return; } while (running.get()) { @@ -849,7 +847,7 @@ private void deliver(String topic, String payload) { } } - private void listenSqlite() { + private void listenOutbox() { long after = 0; boolean initialized = false; while (running.get()) { @@ -861,8 +859,9 @@ private void listenSqlite() { try (var statement = Sql.prepare(c, Sql.query(river.database(), "notification_cursor")); var rows = statement.executeQuery()) { - rows.next(); - return rows.getLong(1); + long cursor = 0; + while (rows.next()) cursor = rows.getLong(1); + return cursor; } }); initialized = true; @@ -993,9 +992,10 @@ private void maintain() throws Exception { } } try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "leader_elect"), + "leader_elect", id, river.database().timestamp(now), river.database().timestamp(expiresAt)); @@ -1144,10 +1144,18 @@ private void clean(Instant now) { config.discardedRetention.isNegative() ? null : river.database().timestamp(now.minus(config.discardedRetention)); - try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "clean_jobs"), cancelled, completed, discarded)) { - statement.executeUpdate(); + if (river.database().dialect() == Database.Dialect.MYSQL) + MySql.clean(river.database(), c, cancelled, completed, discarded); + else { + try (var statement = + Sql.prepare( + c, + Sql.query(river.database(), "clean_jobs"), + cancelled, + completed, + discarded)) { + statement.executeUpdate(); + } } try (var statement = Sql.prepare( @@ -1161,7 +1169,7 @@ private void clean(Instant now) { } private void cleanNotifications(Instant now) { - if (river.database().dialect() != Database.Dialect.SQLITE) return; + if (river.database().dialect() == Database.Dialect.POSTGRES) return; river.transaction( c -> { try (var statement = @@ -1226,13 +1234,12 @@ private void schedule(Instant now) { try { if (!collision) try (var statement = - Sql.prepare( - c, Sql.query(river.database(), "schedule_available"), job.id()); + Sql.mutation(river.database(), c, "schedule_available", job.id()); var rows = statement.executeQuery()) { if (rows.next()) river.notifyInsert(c, rows.getString(1)); } } catch (SQLException error) { - if (!"23505".equals(error.getSQLState())) throw error; + if (!"23505".equals(error.getSQLState()) && error.getErrorCode() != 1062) throw error; c.rollback(savepoint); collision = true; } finally { @@ -1331,9 +1338,10 @@ private void rescue(Instant now, Instant term) { if (active.containsKey(job.id())) continue; var state = rescue.state(); try (var statement = - Sql.prepare( + Sql.mutation( + river.database(), c, - Sql.query(river.database(), "rescue_update"), + "rescue_update", state.value(), river.database().timestamp(state.isFinalized() ? now : null), river.database().timestamp(rescue.scheduled()), diff --git a/java/river/src/main/java/com/riverqueue/Sql.java b/java/river/src/main/java/com/riverqueue/Sql.java index bac27d866..87b0c6a85 100644 --- a/java/river/src/main/java/com/riverqueue/Sql.java +++ b/java/river/src/main/java/com/riverqueue/Sql.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; @@ -12,6 +13,8 @@ final class Sql { private static final Map> QUERIES = Map.of( + Database.Dialect.MYSQL, + load("mysql.sql"), Database.Dialect.POSTGRES, load("postgres.sql"), Database.Dialect.SQLITE, @@ -19,6 +22,63 @@ final class Sql { private Sql() {} + /** Executes a row-returning write within the caller's transaction. */ + static Mutation mutation(Database database, Connection connection, String name, Object... params) + throws SQLException { + return new Mutation(database, connection, name, params); + } + + static final class Mutation implements AutoCloseable { + private final Database database; + private final Connection connection; + private final String name; + private final Object[] params; + private ResultSet rows; + + private Mutation(Database database, Connection connection, String name, Object[] params) { + this.database = database; + this.connection = connection; + this.name = name; + this.params = params; + } + + ResultSet executeQuery() throws SQLException { + if (database.dialect() == Database.Dialect.MYSQL) { + rows = MySql.mutate(database, connection, name, params); + } else { + String sql = query(database, name); + if (name.equals("insert") && database.dialect() == Database.Dialect.POSTGRES) + sql = + sql.replace( + "{duplicate}", + query( + database, + database.usesUniqueNonce(connection) + ? "insert_duplicate_nonce" + : "insert_duplicate_xmax")); + rows = queryRows(connection, sql, params); + } + return rows; + } + + @Override + public void close() throws SQLException { + if (rows != null) rows.close(); + } + } + + static ResultSet queryRows(Connection connection, String sql, Object... params) + throws SQLException { + var statement = prepare(connection, sql, params); + try { + statement.closeOnCompletion(); + return statement.executeQuery(); + } catch (SQLException error) { + statement.close(); + throw error; + } + } + static PreparedStatement prepare(Connection connection, String sql, Object... params) throws SQLException { var statement = connection.prepareStatement(sql); diff --git a/java/river/src/main/java/com/riverqueue/WorkContext.java b/java/river/src/main/java/com/riverqueue/WorkContext.java index e6e254e6e..69a64e3e3 100644 --- a/java/river/src/main/java/com/riverqueue/WorkContext.java +++ b/java/river/src/main/java/com/riverqueue/WorkContext.java @@ -255,9 +255,10 @@ private Job persistCheckpoint(Connection transaction, ObjectNode updates) { transaction, connection -> { try (var statement = - Sql.prepare( + Sql.mutation( + client.database(), connection, - Sql.query(client.database(), "checkpoint"), + "checkpoint", currentStep, Json.encode(updates.isEmpty() ? null : updates), job.id(), diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql new file mode 100644 index 000000000..8a76b2634 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql new file mode 100644 index 000000000..f841bf89a --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/002_initial_schema.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL has no LISTEN/NOTIFY transport, so river_job_notify is omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql new file mode 100644 index 000000000..b544e5000 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. States use VARCHAR with an +-- explicit constraint, so changing them does not require a table rebuild. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql new file mode 100644 index 000000000..5906e5567 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql new file mode 100644 index 000000000..b1e83547d --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..1f368209e --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..82ec71552 --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/migration/mysql/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/java/river/src/main/resources/com/riverqueue/mysql.sql b/java/river/src/main/resources/com/riverqueue/mysql.sql new file mode 100644 index 000000000..37c028fbc --- /dev/null +++ b/java/river/src/main/resources/com/riverqueue/mysql.sql @@ -0,0 +1,298 @@ +-- name: columns +id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states + +-- name: get +SELECT {columns} FROM {schema}river_job WHERE id = ? + +-- name: insert +INSERT INTO {schema}river_job +(args, created_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) +VALUES (CAST(? AS JSON), ?, ?, ?, CAST(? AS JSON), ?, ?, ?, ?, CAST(? AS JSON), ?, ?) +ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) + +-- name: notify +INSERT INTO {schema}river_notification (topic, payload) VALUES (?, ?) + +-- name: migration_insert +INSERT INTO {schema}river_migration (line,version) VALUES ('main',?) + +-- name: migration_insert_legacy +INSERT INTO {schema}river_migration (version) VALUES (?) + +-- name: migration_delete +DELETE FROM {schema}river_migration WHERE line='main' AND version=? + +-- name: migration_delete_legacy +DELETE FROM {schema}river_migration WHERE version=? + +-- name: migration_versions +SELECT version FROM {schema}river_migration WHERE line='main' ORDER BY version + +-- name: migration_versions_legacy +SELECT version FROM {schema}river_migration ORDER BY version + +-- name: list +SELECT {columns} FROM {schema}river_job WHERE {where} ORDER BY {order} LIMIT ? + +-- name: list_deletable +SELECT {columns} FROM {schema}river_job WHERE {where} AND state != 'running' ORDER BY {order} LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: cursor_id +(id {comparison} ?) + +-- name: cursor_null_asc +({field} IS NULL AND id > ?) + +-- name: cursor_null_desc +({field} IS NOT NULL OR id < ?) + +-- name: cursor_time +({field} {comparison} ? OR ({field} = ? AND id {comparison} ?)) + +-- name: cursor_time_nullable +({field} > ? OR ({field} = ? AND id > ?) OR {field} IS NULL) + +-- name: delete +DELETE FROM {schema}river_job WHERE id=? AND state != 'running' + +-- name: delete_filtered +DELETE FROM {schema}river_job WHERE {where} AND id=? AND state != 'running' + +-- name: retry +UPDATE {schema}river_job SET finalized_at=NULL, metadata=JSON_REMOVE(metadata, '$.cancel_attempted_at'), +max_attempts=CASE WHEN attempt=max_attempts THEN max_attempts+1 ELSE max_attempts END, scheduled_at=?, state='available' +WHERE id=? AND state != 'running' AND NOT (state='available' AND scheduled_at < ?) + +-- name: reset_jobs +DELETE FROM {schema}river_job + +-- name: reset_queues +DELETE FROM {schema}river_queue + +-- name: reset_leader +DELETE FROM {schema}river_leader + +-- name: reset_notifications +DELETE FROM {schema}river_notification + +-- name: migration_exists +SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=COALESCE(NULLIF('{schema_name}',''),DATABASE()) AND table_name='river_migration') + +-- name: migration_has_line +SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=COALESCE(NULLIF('{schema_name}',''),DATABASE()) AND table_name='river_migration' AND column_name='line') + +-- name: lock_get +SELECT {columns} FROM {schema}river_job WHERE id=? FOR UPDATE + +-- name: cancel +UPDATE {schema}river_job SET +finalized_at=CASE WHEN state='running' THEN finalized_at ELSE ? END, +metadata=JSON_SET(metadata, '$.cancel_attempted_at', CAST(? AS JSON)), +state=CASE WHEN state='running' THEN state ELSE 'cancelled' END +WHERE id=? AND state NOT IN ('cancelled','completed','discarded') AND finalized_at IS NULL + +-- name: update +UPDATE {schema}river_job SET metadata=CAST(? AS JSON) WHERE id=? + +-- name: filter_tags_all +JSON_CONTAINS(tags, CAST(? AS JSON)) + +-- name: filter_tags_any +JSON_OVERLAPS(tags, CAST(? AS JSON)) + +-- name: filter_ids +id IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value BIGINT PATH '$')) AS wanted) + +-- name: filter_kinds +CAST(kind AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PATH '$')) AS wanted) + +-- name: filter_priorities +priority IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value SMALLINT PATH '$')) AS wanted) + +-- name: filter_queues +CAST(queue AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PATH '$')) AS wanted) + +-- name: filter_states +state IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PATH '$')) AS wanted) + +-- name: queue_pause +UPDATE {schema}river_queue SET updated_at=CASE WHEN paused_at IS NULL THEN ? ELSE updated_at END, paused_at=COALESCE(paused_at, UTC_TIMESTAMP(6)) WHERE (?='*' OR name=CAST(? AS BINARY)) + +-- name: queue_resume +UPDATE {schema}river_queue SET updated_at=CASE WHEN paused_at IS NOT NULL THEN ? ELSE updated_at END, +paused_at=NULL WHERE (?='*' OR name=CAST(? AS BINARY)) + +-- name: queue_heartbeat +INSERT INTO {schema}river_queue (name,created_at,updated_at) VALUES (?,?,?) ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at) + +-- name: leader_get +SELECT leader_id,elected_at FROM {schema}river_leader WHERE name='default' + +-- name: leader_elect +INSERT INTO {schema}river_leader (name,leader_id,elected_at,expires_at) VALUES ('default',?,?,?) ON DUPLICATE KEY UPDATE name=name + +-- name: leader_expire +DELETE FROM {schema}river_leader WHERE expires_at < ? + +-- name: leader_renew +UPDATE {schema}river_leader SET expires_at=? WHERE leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at >= ? + +-- name: leader_resign +DELETE FROM {schema}river_leader WHERE leader_id=CAST(? AS BINARY) AND elected_at=? + +-- name: notifications +SELECT id,topic,payload FROM {schema}river_notification WHERE id>? ORDER BY id LIMIT 256 FOR SHARE + +-- name: queue_get +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue WHERE name=? + +-- name: queue_list +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue ORDER BY name LIMIT ? + +-- name: queue_update +UPDATE {schema}river_queue SET metadata=CAST(? AS JSON), updated_at=? WHERE name=? + +-- name: complete +WITH input AS (SELECT ? AS target_state, ? AS now, ? AS checkpoint_step, CAST(? AS JSON) AS checkpoint_fallback, +CAST(? AS JSON) AS metadata, ? AS state, ? AS finalized_at, ? AS scheduled_at, ? AS attempt, ? AS error_present, CAST(? AS JSON) AS error, +? AS id, ? AS old_attempt, ? AS attempted_at) +UPDATE {schema}river_job j JOIN input i ON j.id=i.id SET +j.errors=CASE WHEN j.state != 'running' OR i.error_present IS NULL THEN j.errors ELSE JSON_ARRAY_APPEND(CASE WHEN j.errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(j.errors)='ARRAY' THEN j.errors ELSE JSON_ARRAY(j.errors) END,'$',i.error) END, +j.finalized_at=CASE WHEN j.state != 'running' THEN j.finalized_at WHEN i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at') THEN i.now ELSE i.finalized_at END, +j.scheduled_at=CASE WHEN j.state != 'running' OR (i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at')) THEN j.scheduled_at ELSE i.scheduled_at END, +j.attempt=CASE WHEN j.state != 'running' OR (i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at')) THEN j.attempt ELSE i.attempt END, +j.state=CASE WHEN j.state != 'running' THEN j.state WHEN i.target_state IN ('available','retryable','scheduled') AND JSON_CONTAINS_PATH(j.metadata,'one','$.cancel_attempted_at') THEN 'cancelled' ELSE i.state END, +j.metadata=i.metadata +WHERE j.attempt=i.old_attempt AND j.attempted_at=i.attempted_at + +-- name: delete_finalized +DELETE FROM {schema}river_job WHERE state IN ('cancelled','completed','discarded') AND finalized_at? ORDER BY id LIMIT 1000 FOR UPDATE SKIP LOCKED + +-- name: cancel_requested +SELECT id FROM {schema}river_job WHERE state='running' AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') + +-- name: notification_cursor +SELECT id FROM {schema}river_notification ORDER BY id FOR UPDATE + +-- name: schedule_select +SELECT id,unique_key FROM {schema}river_job WHERE state IN ('scheduled','retryable') AND scheduled_at<=? ORDER BY priority,scheduled_at,id LIMIT 10000 FOR UPDATE SKIP LOCKED + +-- name: schedule_collision +SELECT id FROM {schema}river_job WHERE id<>? AND unique_key=? AND unique_states IS NOT NULL AND (unique_states & CASE state WHEN 'available' THEN 1 WHEN 'cancelled' THEN 2 WHEN 'completed' THEN 4 WHEN 'discarded' THEN 8 WHEN 'pending' THEN 16 WHEN 'retryable' THEN 32 WHEN 'running' THEN 64 WHEN 'scheduled' THEN 128 ELSE 0 END) != 0 FOR UPDATE + +-- name: schedule_available +UPDATE {schema}river_job SET state='available' WHERE id=? AND state IN ('scheduled','retryable') + +-- name: schedule_discard +UPDATE {schema}river_job SET finalized_at=?, +metadata=JSON_SET(metadata, '$.unique_key_conflict', 'scheduler_discarded'), state='discarded' +WHERE id=? AND state IN ('scheduled','retryable') + +-- name: clean_jobs +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM {schema}river_job WHERE id IN ({ids}) + +-- name: clean_jobs_select +SELECT id FROM {schema}river_job +WHERE (state='cancelled' AND finalized_at'main') + +-- name: inserted +SELECT {columns} FROM {schema}river_job WHERE id = LAST_INSERT_ID() FOR UPDATE + +-- name: delete_rows +SELECT {columns} FROM {schema}river_job WHERE id=? AND state != 'running' FOR UPDATE + +-- name: empty_jobs +SELECT {columns} FROM {schema}river_job WHERE id=? AND false + +-- name: empty_queues +SELECT name,created_at,metadata,paused_at,updated_at FROM {schema}river_queue WHERE name=? AND false + +-- name: filter_metadata +JSON_CONTAINS(metadata, CAST(? AS JSON)) + +-- name: leader_elected +SELECT elected_at FROM {schema}river_leader WHERE leader_id=CAST(? AS BINARY) AND elected_at=? FOR UPDATE + +-- name: scheduled_queue +SELECT queue FROM {schema}river_job WHERE id=? FOR UPDATE + +-- name: migration_lock +SELECT GET_LOCK(CONCAT('river_migrate:',LEFT(SHA2(COALESCE(NULLIF('{schema_name}',''),DATABASE()),256),48)),0) + +-- name: migration_unlock +SELECT RELEASE_LOCK(CONCAT('river_migrate:',LEFT(SHA2(COALESCE(NULLIF('{schema_name}',''),DATABASE()),256),48))) + +-- name: create_schema +CREATE DATABASE IF NOT EXISTS `{schema_name}` + +-- name: claim_select +SELECT id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? +AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL) +ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: claim_known_select +SELECT id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? +AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL) +AND CAST(kind AS BINARY) IN (SELECT value FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PATH '$')) AS wanted) +ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED + +-- name: claim_one +UPDATE {schema}river_job SET attempted_at=?, +attempted_by=JSON_ARRAY_APPEND(CASE WHEN JSON_LENGTH(attempted_by)>=100 THEN JSON_EXTRACT(attempted_by, '$[last-98 to last]') ELSE COALESCE(attempted_by,JSON_ARRAY()) END, '$', ?), +attempt=attempt+1, state='running' WHERE id=? + +-- name: claimed +SELECT {columns} FROM JSON_TABLE(CAST(? AS JSON), '$[*]' COLUMNS(value BIGINT PATH '$')) AS wanted STRAIGHT_JOIN {schema}river_job FORCE INDEX (PRIMARY) ON river_job.id=wanted.value ORDER BY priority,scheduled_at,id + +-- name: filter_metadata_container +({type}='{kind}' AND {children}) + +-- name: filter_metadata_element +(SELECT 1 FROM JSON_TABLE(CASE WHEN {type}='{kind}' THEN {value} END, '$[*]' COLUMNS(value JSON PATH '$')) AS {alias} WHERE {predicate} LIMIT 1) IS NOT NULL + +-- name: filter_metadata_member +(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT(CASE WHEN {type}='{kind}' THEN {value} END, ?), '$' COLUMNS(value JSON PATH '$')) AS {alias} WHERE {predicate} LIMIT 1) IS NOT NULL + +-- name: filter_metadata_scalar +(SELECT 1 FROM JSON_TABLE(CAST(? AS JSON), '$' COLUMNS(value JSON PATH '$')) AS wanted +WHERE {type} NOT IN ('object','array') AND ({value} IS NULL AND wanted.value IS NULL OR JSON_CONTAINS({value}, wanted.value)) LIMIT 1) IS NOT NULL + +-- name: checkpoint +UPDATE {schema}river_job SET metadata=JSON_SET(metadata, '$."river:resumable_step"', ?, '$."river:resumable_cursor"', CAST(? AS JSON)) +WHERE id=? AND state='running' AND attempt=? AND attempted_at=? diff --git a/java/river/src/test/java/com/riverqueue/ClientTest.java b/java/river/src/test/java/com/riverqueue/ClientTest.java index dd0ce3daf..96f27d559 100644 --- a/java/river/src/test/java/com/riverqueue/ClientTest.java +++ b/java/river/src/test/java/com/riverqueue/ClientTest.java @@ -12,6 +12,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; @@ -48,6 +49,50 @@ void applicationTransactionOwnsCommitAndRollback() throws Exception { } } + @Test + @Tag("mysql") + void cleanupSkipsLockedJobs() throws Exception { + var client = database(); + var database = client.database(); + org.junit.jupiter.api.Assumptions.assumeTrue(database.dialect() == Database.Dialect.MYSQL); + var locked = client.insert(EMAIL, new Email("locked@example.com")).job(); + var unlocked = client.insert(EMAIL, new Email("unlocked@example.com")).job(); + for (int i = 0; i < 999; i++) client.insert(EMAIL, new Email("expired@example.com")); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET state='completed', finalized_at='1970-01-01 00:00:00'"); + } + Client.Transaction clean = + connection -> { + try (var statement = connection.createStatement()) { + statement.execute("SET SESSION innodb_lock_wait_timeout=2"); + } + return MySql.clean( + database, + connection, + database.timestamp(NOW), + database.timestamp(NOW), + database.timestamp(NOW)); + }; + client.transaction( + connection -> { + try (var statement = + Sql.prepare(connection, Sql.query(database, "lock_get"), locked.id()); + var rows = statement.executeQuery()) { + assertTrue(rows.next()); + assertEquals(1000, client.transaction(clean)); + } + assertThrows(RiverException.class, () -> client.jobs().get(unlocked.id())); + assertEquals(locked.id(), client.jobs().get(locked.id()).id()); + return null; + }); + assertEquals(1, client.transaction(clean)); + assertTrue(client.jobs().list(JobQuery.all()).jobs().isEmpty()); + } + @Test void completionMetadataReplacesTopLevelKeys() { var client = database(); @@ -74,9 +119,10 @@ void completionMetadataReplacesTopLevelKeys() { client.transaction( connection -> { try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", database.timestamp(NOW), "worker", "default", @@ -185,9 +231,10 @@ void delayedCompletionPreservesARescheduledJob(Job.State state) { client.transaction( connection -> { try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", database.timestamp(NOW), "old-worker", "default", @@ -486,6 +533,10 @@ void typedGetAndMixedBatchRetainArgumentKinds() throws Exception { @Test void checkedHookFailureRollsBackInsideCallerTransaction() throws Exception { var base = database(); + try (var connection = base.database().connection(); + var statement = connection.createStatement()) { + statement.execute("CREATE TABLE " + base.database().prefix() + "hook_write (id INTEGER)"); + } var river = base.withHooks( new HookInsertBegin() { @@ -495,7 +546,7 @@ public void insertBegin( throws java.sql.SQLException { try (var statement = connection.createStatement()) { statement.execute( - "CREATE TABLE " + base.database().prefix() + "hook_write (id INTEGER)"); + "INSERT INTO " + base.database().prefix() + "hook_write VALUES (1)"); } throw new java.sql.SQLException("hook rejected"); } @@ -510,15 +561,12 @@ public void insertBegin( connection, List.of(EMAIL.submission(new Email("rejected@example.com"))))); assertEquals(RiverException.Code.DATABASE, failure.code()); assertInstanceOf(java.sql.SQLException.class, failure.getCause()); - try (var rows = - connection - .getMetaData() - .getTables( - null, - river.database().schema().isEmpty() ? null : river.database().schema(), - "hook_write", - null)) { - assertFalse(rows.next()); + try (var statement = connection.createStatement(); + var rows = + statement.executeQuery( + "SELECT count(*) FROM " + base.database().prefix() + "hook_write")) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1)); } var kept = base.insert(connection, EMAIL, new Email("kept@example.com")).job(); connection.commit(); @@ -765,6 +813,41 @@ void uniqueByFieldsSelectsJsonNamesWithoutSplittingDots() { assertNotEquals(first.job().id(), different.job().id()); } + @ParameterizedTest + @Tag("mysql") + @ValueSource(strings = {"inactive_existing", "inactive_incoming"}) + void unrelatedUniqueConflictsAreRejectedWhenReinserting(String mode) throws Exception { + var client = database(); + var database = client.database(); + org.junit.jupiter.api.Assumptions.assumeTrue(database.dialect() == Database.Dialect.MYSQL); + var inserted = + client + .insert( + EMAIL, + new Email("unique@example.com"), + InsertOptions.builder().unique(Unique.args()).build()) + .job(); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "CREATE UNIQUE INDEX custom_kind ON " + database.prefix() + "river_job(kind)"); + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET state='running', unique_states=" + + (mode.equals("inactive_incoming") ? "64" : "NULL")); + } + var original = client.jobs().get(inserted.id()); + // Reinsertion uses available; either its key or the existing key is inactive. + var error = + assertThrows( + RiverException.class, + () -> client.transaction(connection -> client.driver().reinsert(connection, original))); + assertTrue(error.getMessage().contains("unrelated unique constraint")); + assertEquals(original, client.jobs().get(original.id())); + assertEquals(1, client.jobs().list(JobQuery.all()).jobs().size()); + } + record Report(int accountId) {} record Email(String address) {} diff --git a/java/river/src/test/java/com/riverqueue/MySqlTest.java b/java/river/src/test/java/com/riverqueue/MySqlTest.java new file mode 100644 index 000000000..20097f781 --- /dev/null +++ b/java/river/src/test/java/com/riverqueue/MySqlTest.java @@ -0,0 +1,102 @@ +package com.riverqueue; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +@Tag("mysql") +@EnabledIfSystemProperty(named = "river.test.database", matches = "mysql") +class MySqlTest { + @TempDir Path directory; + @RegisterExtension final TestDatabase databases = new TestDatabase(); + + @Test + void matchesKindsAndQueuesExactly() throws Exception { + var database = databases.open(directory.resolve("unused")); + var client = new Client(database); + var type = JobType.of("message", String.class); + var wanted = client.insert(type, "first").job(); + var padded = client.insert(type, "second").job(); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "UPDATE " + + database.prefix() + + "river_job SET kind='message ', queue='default ' WHERE id=" + + padded.id()); + } + assertEquals( + List.of(wanted.id()), + client.jobs().list(JobQuery.builder().kinds("message").build()).jobs().stream() + .map(Job::id) + .toList()); + assertEquals( + List.of(wanted.id()), + client.jobs().list(JobQuery.builder().queues("default").build()).jobs().stream() + .map(Job::id) + .toList()); + } + + @Test + void preservesExactQueueNames() throws Exception { + var client = new Client(databases.open(directory.resolve("unused"))); + var names = java.util.List.of("queue", "queue ", "QUEUE", "cafĂ©", "cafe"); + for (var name : names) { + client.transaction( + connection -> { + try (var statement = + Sql.prepare( + connection, + Sql.query(client.database(), "queue_heartbeat"), + name, + client.database().timestamp(java.time.Instant.now()), + client.database().timestamp(java.time.Instant.now()))) { + statement.executeUpdate(); + } + return null; + }); + } + for (var name : names) assertEquals(name, client.queues().get(name).name()); + } + + @Test + void migrationRoundTrip() { + var database = databases.open(directory.resolve("unused")); + var migrator = new Migrator(database); + assertTrue(migrator.migrate().valid()); + assertEquals( + List.of(8, 7, 6, 5, 4, 3, 2, 1), + migrator + .migrate(Migrator.Direction.DOWN, Migrator.Options.defaults().targetVersion(0)) + .applied()); + assertTrue(migrator.list().stream().noneMatch(Migrator.Status::applied)); + assertTrue(migrator.migrate().valid()); + } + + @Test + void largeIdsRemainDistinct() throws Exception { + var database = databases.open(directory.resolve("unused")); + try (var connection = database.connection(); + var statement = connection.createStatement()) { + statement.execute( + "ALTER TABLE " + database.prefix() + "river_job AUTO_INCREMENT=9007199254740993"); + } + var client = new Client(database); + var type = JobType.of("message", String.class); + var first = client.insert(type, "first").job(); + var second = client.insert(type, "second").job(); + assertEquals(9007199254740993L, first.id()); + assertEquals(first.id() + 1, second.id()); + assertEquals( + List.of(second.id()), + client.jobs().list(JobQuery.builder().ids(second.id()).build()).jobs().stream() + .map(Job::id) + .toList()); + } +} diff --git a/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java b/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java index 5e845a651..ba9c262b8 100644 --- a/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java +++ b/java/river/src/test/java/com/riverqueue/PeerCancellationTest.java @@ -78,9 +78,10 @@ public JsonNode decode(Job job) { var jobs = new ArrayList(); var now = database.timestamp(Instant.now()); try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", now, "batch", "peers", @@ -194,9 +195,10 @@ public void work(WorkContext context) throws Exception { var jobs = new ArrayList(); var now = database.timestamp(Instant.now()); try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", now, "batch", "peers", diff --git a/java/river/src/test/java/com/riverqueue/PeerDecodingTest.java b/java/river/src/test/java/com/riverqueue/PeerDecodingTest.java index f9cf4ed5c..01650616a 100644 --- a/java/river/src/test/java/com/riverqueue/PeerDecodingTest.java +++ b/java/river/src/test/java/com/riverqueue/PeerDecodingTest.java @@ -92,9 +92,10 @@ public JsonNode decode(Job job) { var jobs = new ArrayList(); var now = database.timestamp(Instant.now()); try (var statement = - Sql.prepare( + Sql.mutation( + database, connection, - Sql.query(database, "claim"), + "claim", now, "batch", "peers", diff --git a/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java b/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java index 169928955..4459a587c 100644 --- a/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java +++ b/java/river/src/test/java/com/riverqueue/ProtocolStorageTest.java @@ -97,7 +97,11 @@ public boolean clean(Client client, Instant now) { "UPDATE " + database.prefix() + "river_job SET state = 'running', attempt = 3, attempted_at = ?, errors = " - + (postgres ? "ARRAY[?::jsonb]" : "json_array(json(?))") + + (switch (database.dialect()) { + case MYSQL -> "JSON_ARRAY(CAST(? AS JSON))"; + case POSTGRES -> "ARRAY[?::jsonb]"; + case SQLITE -> "json_array(json(?))"; + }) + " WHERE id = ?", database.timestamp(Instant.EPOCH), Json.encode(fixture.required("attempt_error")), @@ -120,7 +124,7 @@ public boolean clean(Client client, Instant now) { .errorHandler(errors::add) .retryPolicy( job -> { - if (!postgres) { + if (database.dialect() == Database.Dialect.SQLITE) { // Retry selection runs after the read transaction closes. Keep the normal // busy timeout for unrelated worker writes; a transaction held by this // callback would stay locked until the timeout expires. @@ -140,13 +144,13 @@ public boolean clean(Client client, Instant now) { .start(); var subscription = workers.client().subscribe(completed::complete, Client.EventKind.JOB_COMPLETED)) { - if (!postgres) + if (database.dialect() == Database.Dialect.SQLITE) assertNull( writeFailure.get(10, TimeUnit.SECONDS), "Rescue callbacks must not hold the writer"); maintained.get(5, TimeUnit.SECONDS); var rescued = client.jobs().get(inserted.id()); assertEquals(Job.State.RETRYABLE, rescued.state()); - if (!postgres) assertEquals(2, rescued.priority()); + if (database.dialect() == Database.Dialect.SQLITE) assertEquals(2, rescued.priority()); assertEquals(3, rescued.metadata().required(rescueKey).asInt()); assertEquals(2, rescued.errors().size()); assertEquals(fixture.required("attempt_error"), Json.tree(rescued.errors().getFirst())); diff --git a/java/river/src/test/java/com/riverqueue/RunHandleTest.java b/java/river/src/test/java/com/riverqueue/RunHandleTest.java index 8bd7dc7a3..8697ca53e 100644 --- a/java/river/src/test/java/com/riverqueue/RunHandleTest.java +++ b/java/river/src/test/java/com/riverqueue/RunHandleTest.java @@ -672,9 +672,10 @@ void goResignationWakesLeadershipBeforeNextPoll() throws Exception { connection -> { var now = Instant.now(); try (var statement = - Sql.prepare( + Sql.mutation( + client.database(), connection, - Sql.query(client.database(), "leader_elect"), + "leader_elect", leaderId, client.database().timestamp(now), client.database().timestamp(now.plusSeconds(3600))); diff --git a/java/river/src/test/java/com/riverqueue/SchedulingTest.java b/java/river/src/test/java/com/riverqueue/SchedulingTest.java index 73355dfeb..c5d94fb8a 100644 --- a/java/river/src/test/java/com/riverqueue/SchedulingTest.java +++ b/java/river/src/test/java/com/riverqueue/SchedulingTest.java @@ -286,7 +286,9 @@ public void afterInsert( assertNotNull(second); assertEquals(java.util.Set.of("periodic", "scheduled"), java.util.Set.of(first, second)); assertFalse(errors.isEmpty()); - assertTrue(errors.stream().allMatch(error -> "periodic failed".equals(error.getMessage()))); + assertTrue( + errors.stream().allMatch(error -> "periodic failed".equals(error.getMessage())), + () -> "Unexpected errors: " + errors); } } diff --git a/java/river/src/test/java/com/riverqueue/TestDatabase.java b/java/river/src/test/java/com/riverqueue/TestDatabase.java index 365efd588..b1c2a9f1c 100644 --- a/java/river/src/test/java/com/riverqueue/TestDatabase.java +++ b/java/river/src/test/java/com/riverqueue/TestDatabase.java @@ -16,7 +16,10 @@ public void afterEach(ExtensionContext context) throws Exception { for (var database : databases) try (var connection = database.connection(); var statement = connection.createStatement()) { - statement.execute("DROP SCHEMA IF EXISTS \"" + database.schema() + "\" CASCADE"); + statement.execute( + database.dialect() == Database.Dialect.MYSQL + ? "DROP DATABASE IF EXISTS `" + database.schema() + "`" + : "DROP SCHEMA IF EXISTS \"" + database.schema() + "\" CASCADE"); } } @@ -24,14 +27,14 @@ Database open(Path file) { String url = System.getenv("RIVER_TEST_DATABASE_URL"); String backend = System.getProperty("river.test.database", url == null ? "sqlite" : "postgres"); if (backend.equals("sqlite")) return sqlite(file); - if (!backend.equals("postgres")) + if (!backend.equals("postgres") && !backend.equals("mysql")) throw new IllegalArgumentException("Unknown test backend: " + backend); if (url == null || url.isBlank()) throw new IllegalStateException("RIVER_TEST_DATABASE_URL is required for Postgres tests"); var database = Database.connect(url).withSchema("java_" + UUID.randomUUID().toString().replace("-", "")); - if (database.dialect() != Database.Dialect.POSTGRES) - throw new IllegalArgumentException("RIVER_TEST_DATABASE_URL must select Postgres"); + if (!database.dialect().name().equalsIgnoreCase(backend)) + throw new IllegalArgumentException("RIVER_TEST_DATABASE_URL must select " + backend); databases.add(database); new Migrator(database).migrate(); return database; diff --git a/job_list_params.go b/job_list_params.go index a42bbe9c9..099269cb9 100644 --- a/job_list_params.go +++ b/job_list_params.go @@ -179,6 +179,8 @@ type JobListParams struct { where []dblist.WherePredicate } +const jobListMetadataPredicatePostgres = `metadata @> @metadata_fragment::jsonb` + // NewJobListParams creates a new JobListParams to return available jobs sorted // by time in ascending order, returning 100 jobs at most. func NewJobListParams() *JobListParams { @@ -357,9 +359,9 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { // first descending. case p.after.time.IsZero(): if sortOrder == dblist.SortOrderAsc { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" IS NULL AND "id" > @after_id)`, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s IS NULL AND id > @after_id)`, timeField)}) } else { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" IS NOT NULL OR "id" < @after_id)`, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s IS NOT NULL OR id < @after_id)`, timeField)}) } default: @@ -367,11 +369,11 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { if sortOrder == dblist.SortOrderAsc { var orNull string if timeFieldNullable { - orNull = fmt.Sprintf(` OR "%s" IS NULL`, timeField) + orNull = fmt.Sprintf(` OR %s IS NULL`, timeField) } - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" > @cursor_time OR ("%s" = @cursor_time AND "id" > @after_id)%s)`, timeField, timeField, orNull)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s > @cursor_time OR (%s = @cursor_time AND id > @after_id)%s)`, timeField, timeField, orNull)}) } else { - where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`("%s" < @cursor_time OR ("%s" = @cursor_time AND "id" < @after_id))`, timeField, timeField)}) + where = append(where, dblist.WherePredicate{NamedArgs: namedArgs, SQL: fmt.Sprintf(`(%s < @cursor_time OR (%s = @cursor_time AND id < @after_id))`, timeField, timeField)}) } } } @@ -391,6 +393,22 @@ func (p *JobListParams) toDBParams() (*dblist.JobListParams, error) { }, nil } +func (p *JobListParams) withMetadataPredicateSQL(buildFunc func(string, string, string) (string, map[string]any, error)) (*JobListParams, error) { + paramsCopy := p.copy() + for i := range paramsCopy.where { + predicate := ¶msCopy.where[i] + if predicate.SQL == jobListMetadataPredicatePostgres { + fragment, _ := predicate.NamedArgs["metadata_fragment"].(string) + var err error + predicate.SQL, predicate.NamedArgs, err = buildFunc("metadata", "metadata_fragment", fragment) + if err != nil { + return nil, err + } + } + } + return paramsCopy, nil +} + // After returns an updated filter set that will only return jobs // after the given cursor. func (p *JobListParams) After(cursor *JobListCursor) *JobListParams { @@ -452,7 +470,7 @@ func (p *JobListParams) Metadata(json string) *JobListParams { paramsCopy.metadataCalled = true paramsCopy.where = append(paramsCopy.where, dblist.WherePredicate{ NamedArgs: map[string]any{"metadata_fragment": json}, - SQL: `metadata @> @metadata_fragment::jsonb`, + SQL: jobListMetadataPredicatePostgres, }) return paramsCopy } diff --git a/job_list_params_test.go b/job_list_params_test.go index 0e741ed39..667566dbd 100644 --- a/job_list_params_test.go +++ b/job_list_params_test.go @@ -429,7 +429,7 @@ func Test_JobListParams_toDBParams_CustomConditions(t *testing.T) { Where("state = @other_state OR finalized_at IS NULL", NamedArgs{"other_state": "completed"}). After(&JobListCursor{id: 42, time: cursorTime}) driverParams := driverParamsFunc(t, bundle) - const cursorSQL = `("finalized_at" < @cursor_time OR ("finalized_at" = @cursor_time AND "id" < @after_id))` + const cursorSQL = `(finalized_at < @cursor_time OR (finalized_at = @cursor_time AND id < @after_id))` require.Equal(t, "state = any(@state)\n AND state = @other_state OR finalized_at IS NULL\n AND "+cursorSQL, driverParams.WhereClause) require.Equal(t, map[string]any{ "after_id": int64(42), @@ -530,29 +530,29 @@ func Test_JobListParams_toDBParamsNullableTimeField(t *testing.T) { name: "AttemptedAtAsc", params: NewJobListParams().States(rivertype.JobStateRunning).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "attempted_at ASC NULLS LAST, id ASC", - wantAfterNull: `("attempted_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("attempted_at" > @cursor_time OR ("attempted_at" = @cursor_time AND "id" > @after_id) OR "attempted_at" IS NULL)`, + wantAfterNull: `(attempted_at IS NULL AND id > @after_id)`, + wantAfterTime: `(attempted_at > @cursor_time OR (attempted_at = @cursor_time AND id > @after_id) OR attempted_at IS NULL)`, }, { name: "AttemptedAtDesc", params: NewJobListParams().States(rivertype.JobStateRunning).OrderBy(JobListOrderByTime, SortOrderDesc), wantOrderBy: "attempted_at DESC NULLS FIRST, id DESC", - wantAfterNull: `("attempted_at" IS NOT NULL OR "id" < @after_id)`, - wantAfterTime: `("attempted_at" < @cursor_time OR ("attempted_at" = @cursor_time AND "id" < @after_id))`, + wantAfterNull: `(attempted_at IS NOT NULL OR id < @after_id)`, + wantAfterTime: `(attempted_at < @cursor_time OR (attempted_at = @cursor_time AND id < @after_id))`, }, { name: "FinalizedAtMixedStatesAsc", params: NewJobListParams().States(rivertype.JobStateCompleted, rivertype.JobStateAvailable).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterNull: `("finalized_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id) OR "finalized_at" IS NULL)`, + wantAfterNull: `(finalized_at IS NULL AND id > @after_id)`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id) OR finalized_at IS NULL)`, }, { name: "FinalizedAtMixedStatesDesc", params: NewJobListParams().States(rivertype.JobStateCompleted, rivertype.JobStateAvailable).OrderBy(JobListOrderByTime, SortOrderDesc), wantOrderBy: "finalized_at DESC NULLS FIRST, id DESC", - wantAfterNull: `("finalized_at" IS NOT NULL OR "id" < @after_id)`, - wantAfterTime: `("finalized_at" < @cursor_time OR ("finalized_at" = @cursor_time AND "id" < @after_id))`, + wantAfterNull: `(finalized_at IS NOT NULL OR id < @after_id)`, + wantAfterTime: `(finalized_at < @cursor_time OR (finalized_at = @cursor_time AND id < @after_id))`, }, { // An ungrouped OR can include available jobs despite the completed @@ -560,20 +560,20 @@ func Test_JobListParams_toDBParamsNullableTimeField(t *testing.T) { name: "FinalizedAtWithCondition", params: NewJobListParams().States(rivertype.JobStateCompleted).OrderBy(JobListOrderByTime, SortOrderAsc).Where("state = 'available' OR state = 'completed'"), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterNull: `("finalized_at" IS NULL AND "id" > @after_id)`, - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id) OR "finalized_at" IS NULL)`, + wantAfterNull: `(finalized_at IS NULL AND id > @after_id)`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id) OR finalized_at IS NULL)`, }, { name: "FinalizedStates", params: NewJobListParams().OrderBy(JobListOrderByFinalizedAt, SortOrderAsc), wantOrderBy: "finalized_at ASC NULLS LAST, id ASC", - wantAfterTime: `("finalized_at" > @cursor_time OR ("finalized_at" = @cursor_time AND "id" > @after_id))`, + wantAfterTime: `(finalized_at > @cursor_time OR (finalized_at = @cursor_time AND id > @after_id))`, }, { name: "ScheduledAt", params: NewJobListParams().States(rivertype.JobStateAvailable, rivertype.JobStateCompleted).OrderBy(JobListOrderByTime, SortOrderAsc), wantOrderBy: "scheduled_at ASC NULLS LAST, id ASC", - wantAfterTime: `("scheduled_at" > @cursor_time OR ("scheduled_at" = @cursor_time AND "id" > @after_id))`, + wantAfterTime: `(scheduled_at > @cursor_time OR (scheduled_at = @cursor_time AND id > @after_id))`, }, } { t.Run(tt.name, func(t *testing.T) { diff --git a/js/CHANGELOG.md b/js/CHANGELOG.md index 095fca8c7..fdc11005f 100644 --- a/js/CHANGELOG.md +++ b/js/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `@riverqueue/driver-mysql` with worker execution, transactional job operations, outbox notifications, maintenance, and API/CLI migrations. + ## [0.3.1] - 2026-11-05 ### Fixed diff --git a/js/README.md b/js/README.md index 481b32820..89bcea747 100644 --- a/js/README.md +++ b/js/README.md @@ -1,6 +1,6 @@ # River for JavaScript and TypeScript -River is a fast, reliable background job system backed by Postgres or SQLite. +River is a fast, reliable background job system backed by MySQL, Postgres, or SQLite. This implementation runs on Node.js and shares River's database protocol with River's other language implementations, so services can insert and work the same jobs in the same database. @@ -132,13 +132,15 @@ River never closes the pool you give it and never migrates on its own. Web servers that only insert jobs construct the same `Client` without `queues` or `workers` and never call `start()`. +For MySQL, use [`@riverqueue/driver-mysql`](./driver/mysql/README.md) with `mysql2`. + For SQLite, install `@riverqueue/driver-sqlite` instead of the Postgres packages; it uses Node's built-in `node:sqlite`. See the [SQLite driver](./driver/sqlite/README.md). ## Transactions and other features -[Insert jobs in an application transaction](./docs/README.md#transactions) so the job and its related database changes commit or roll back together. River supports transactions with node-postgres, Prisma, and SQLite. +[Insert jobs in an application transaction](./docs/README.md#transactions) so the job and its related database changes commit or roll back together. River supports transactions with mysql2, node-postgres, Prisma, and SQLite. - [Bulk insertion](./docs/README.md#batches) and [unique jobs](./docs/README.md#unique-jobs). - [Retries, timeouts, cancellation, and snoozing](./docs/errors-and-retries.md). @@ -153,10 +155,11 @@ packages; it uses Node's built-in `node:sqlite`. See the | ---------------------------- | ------------------------------------------------------------------- | | `riverqueue` | Job definitions, insertion, workers, runtime, queries, and events | | `@riverqueue/driver-knex` | Insert jobs inside Knex transactions on Postgres | +| `@riverqueue/driver-mysql` | MySQL through `mysql2` | | `@riverqueue/driver-pg` | Postgres through `node-postgres` | | `@riverqueue/driver-prisma` | Insert jobs inside Prisma transactions | | `@riverqueue/driver-sqlite` | SQLite through Node's built-in `node:sqlite` | -| `@riverqueue/migrate` | Postgres and SQLite migrations | +| `@riverqueue/migrate` | MySQL, Postgres, and SQLite migrations | | `@riverqueue/worker-threads` | Run CPU-bound handlers on worker threads | | `@riverqueue/test` | Test helpers for producers and workers | | `@riverqueue/cli` | The `riverqueue` command: migrations, benchmarks, and a 0.1 codemod | diff --git a/js/cli/package.json b/js/cli/package.json index e8cda4c45..a613140b7 100644 --- a/js/cli/package.json +++ b/js/cli/package.json @@ -49,6 +49,7 @@ "provenance": true }, "dependencies": { + "mysql2": "^3.24.4", "@js-temporal/polyfill": "^0.5.1", "@riverqueue/driver-pg": "workspace:0.3.1", "@riverqueue/migrate": "workspace:0.3.1", diff --git a/js/cli/src/database.ts b/js/cli/src/database.ts index 942ce05df..5b17a1742 100644 --- a/js/cli/src/database.ts +++ b/js/cli/src/database.ts @@ -16,6 +16,7 @@ export type DatabaseLocation = readonly backend: "postgres"; readonly connectionString: string | undefined; } + | { readonly backend: "mysql"; readonly connectionString: string } | { readonly backend: "sqlite"; readonly path: string | URL }; // Match River's Go CLI: bound statements so a stuck lock can't hang a @@ -65,6 +66,8 @@ export function parseDatabaseUrl( const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(url)?.[1]?.toLowerCase(); switch (scheme) { + case "mysql": + return { backend: "mysql", connectionString: url }; case "postgres": case "postgresql": return { backend: "postgres", connectionString: url }; @@ -83,7 +86,7 @@ export function parseDatabaseUrl( } default: throw new UsageError( - "--database-url must start with postgres://, postgresql://, or sqlite://", + "--database-url must start with postgres://, postgresql://, mysql://, or sqlite://", command ); } diff --git a/js/cli/src/migrate-commands.ts b/js/cli/src/migrate-commands.ts index 9d2513a42..a2ceaeeb4 100644 --- a/js/cli/src/migrate-commands.ts +++ b/js/cli/src/migrate-commands.ts @@ -1,3 +1,4 @@ +import { createPool } from "mysql2/promise"; import { quoteIdentifier } from "riverqueue/unstable-driver"; import { createMigrator, @@ -33,7 +34,7 @@ const TEMPLATE_SCHEMA = "/* TEMPLATE: schema */"; const DATABASE_URL_OPTION: OptionSpec = { description: - "Database to use: postgres://..., postgresql://..., or sqlite://PATH " + + "Database to use: mysql://..., postgres://..., postgresql://..., or sqlite://PATH " + "(defaults to PG* environment variables when PGDATABASE is set)", type: "string", valueName: "URL", @@ -47,7 +48,7 @@ const LINE_OPTION: OptionSpec = { const SCHEMA_OPTION: OptionSpec = { description: - "Postgres schema containing River's tables (default: the search_path)", + "MySQL database or Postgres schema containing River's tables (default: the connection's current schema)", type: "string", valueName: "NAME", }; @@ -319,7 +320,18 @@ async function runMigrateGet( }); const direction = down ? "down" : "up"; - const prefix = schema === undefined ? "" : `${quoteIdentifier(schema)}.`; + if ( + backend === "mysql" && + schema !== undefined && + !/^[A-Za-z_][A-Za-z0-9_]{0,45}$/.test(schema) + ) + throw new UsageError("Invalid MySQL schema name", command); + const prefix = + schema === undefined + ? "" + : backend === "mysql" + ? `\`${schema}\`.` + : `${quoteIdentifier(schema)}.`; let printed = false; for (const migration of selected) { if (excluded.has(migration.version)) continue; @@ -373,6 +385,25 @@ async function withMigrator( } } + if (location.backend === "mysql") { + if (statementTimeoutMs !== undefined) + throw new UsageError( + "--statement-timeout only applies to Postgres", + command + ); + const pool = createPool(location.connectionString); + try { + return await run( + createMigrator( + { backend: "mysql", pool, schema }, + { line, migrations: lineMigrations(context, line, "mysql") } + ) + ); + } finally { + await pool.end(); + } + } + const pool = openPostgresPool(location, { max: 2, statementTimeoutMs }); try { return await run( @@ -459,9 +490,10 @@ function sqlDialect( if (url === undefined) return "postgres"; const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(url)?.[1]?.toLowerCase(); if (scheme === "postgres" || scheme === "postgresql") return "postgres"; + if (scheme === "mysql") return "mysql"; if (scheme === "sqlite") return "sqlite"; throw new UsageError( - "--database-url must start with postgres://, postgresql://, or sqlite://", + "--database-url must start with postgres://, postgresql://, mysql://, or sqlite://", command ); } diff --git a/js/cli/src/run.test.ts b/js/cli/src/run.test.ts index ffff46908..5ae16b532 100644 --- a/js/cli/src/run.test.ts +++ b/js/cli/src/run.test.ts @@ -163,7 +163,7 @@ describe("argument errors", () => { it.each([ [ - ["migrate-up", "--database-url", "mysql://localhost/river"], + ["migrate-up", "--database-url", "oracle://localhost/river"], "--database-url must start with postgres://", ], [ diff --git a/js/docs/development.md b/js/docs/development.md index 5cc952ccc..d8457a8ee 100644 --- a/js/docs/development.md +++ b/js/docs/development.md @@ -217,6 +217,14 @@ both drivers' notification adapters without a Postgres server. The two raw JSON unique-key cases involving duplicate keys or integer-key insertion order are Rust-only because JavaScript objects cannot preserve them. +`TEST_MYSQL_URL=mysql://root@localhost/ make test/js/conformance/mysql` runs +the MySQL storage checks against the same Go fixtures: state masks and +uniqueness conflicts, attempt errors, rescue/output metadata, exact snooze +counters, and notification emission and reception. Each test owns a temporary +database created with the canonical migrations. The shared Conformance workflow +runs both fixture targets when Go changes; the MySQL integration jobs include +these cases too. + ## Preparing a release Run this section's commands from the repository root. The nine publishable packages are `js/package.json` (`riverqueue`), the four drivers under `js/driver/*`, and `js/{migrate,worker-threads,test,cli}`. They share one version, independently of Go and Rust. Examples are private and stay at `0.0.0`. `VERSION` has no leading `v`; JavaScript Git tags use `js/vX.Y.Z`. diff --git a/js/driver/mysql/README.md b/js/driver/mysql/README.md new file mode 100644 index 000000000..2c07aa034 --- /dev/null +++ b/js/driver/mysql/README.md @@ -0,0 +1,56 @@ +# MySQL driver + +`@riverqueue/driver-mysql` provides River's client, worker runtime, and migrations +on MySQL 8.0 and later. Use the latest patch release of your MySQL version. +Jobs use the same schema and protocol as River for Go. + +```sh +npm install riverqueue @riverqueue/driver-mysql @riverqueue/migrate mysql2 +``` + +```ts +import { createPool } from "mysql2/promise"; +import { Client, defineJob, Workers } from "riverqueue"; +import { MySqlDriver } from "@riverqueue/driver-mysql"; +import { createMigrator } from "@riverqueue/migrate"; + +const pool = createPool("mysql://localhost/my_app"); +const driver = new MySqlDriver(pool); +await createMigrator(driver).migrateUp(); + +const message = defineJob({ kind: "message" }); +const workers = new Workers().add(message, () => + console.log("Message received") +); +const client = new Client(driver, { + queues: { default: { maxWorkers: 10 } }, + workers, +}); +const run = await client.start(); +await client.insert(message, {}); +await run.stop(); +await pool.end(); +``` + +Run migrations during deployment before starting workers. The CLI accepts +`--database-url mysql://localhost/my_app`. MySQL DDL commits implicitly: a +migration failure can leave partially applied DDL, while migration history is +recorded only after a version succeeds. Inspect and repair the failed version +before retrying. A named lock serializes migrators for the same database. + +Pass `{ schema: "background_jobs" }` to the driver to keep River's tables in a +separate MySQL database. River never closes your pool. Its own transactions use +READ COMMITTED; caller transactions retain their configured isolation level. + +To insert within an application transaction, check out a connection, call +`beginTransaction()`, and pass it as `{ tx }` to River. Commit or roll back the +connection yourself, then release it. Worker contexts also support completing +jobs in that transaction through `completeTx`. + +Claims use `FOR UPDATE SKIP LOCKED`. Notifications use the durable +`river_notification` outbox, including cancellation, queue controls, and +leadership resignation requests. Job IDs remain `bigint` and timestamps retain +microsecond precision regardless of the pool's JSON, date, or bigint options. + +See the shared [MySQL design notes](../../../docs/development.md#mysql-design-and-operation) for JSON numeric limits, +exact string comparisons, and server packet-size requirements. diff --git a/js/driver/mysql/etc/driver-mysql.api.md b/js/driver/mysql/etc/driver-mysql.api.md new file mode 100644 index 000000000..c36f922fb --- /dev/null +++ b/js/driver/mysql/etc/driver-mysql.api.md @@ -0,0 +1,33 @@ +# API report: `@riverqueue/driver-mysql` + + + +This report contains declarations and TSDoc for names exported by the +package entry point. Private members, unexported implementation +declarations, and file layout are omitted. + +## `MySqlDriver` + +```ts +/** + * MySQL 8.0.17+ backend using a caller-owned mysql2 pool. Pass an open + * connection as `{ tx }` to join an application transaction. River never + * closes the pool or commits a caller's transaction. + */ +export declare class MySqlDriver { + readonly "~river"?: { + readonly capability: "runtime"; + readonly transaction: PoolConnection; + }; + constructor(pool: Pool, options?: MySqlDriverOptions); +} +``` + +## `MySqlDriverOptions` + +```ts +/** MySQL database containing River's tables; defaults to the pool's database. */ +export interface MySqlDriverOptions { + readonly schema?: string; +} +``` diff --git a/js/driver/mysql/package.json b/js/driver/mysql/package.json new file mode 100644 index 000000000..8be7ffb85 --- /dev/null +++ b/js/driver/mysql/package.json @@ -0,0 +1,60 @@ +{ + "name": "@riverqueue/driver-mysql", + "version": "0.3.1", + "description": "MySQL backend for River's TypeScript runtime.", + "type": "module", + "sideEffects": false, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "engines": { + "node": ">=24" + }, + "files": [ + "dist", + "src", + "!src/**/*.test.ts", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "node ../../node_modules/typescript/bin/tsc && node ../../scripts/copy-license.mjs", + "clean": "rm -rf dist", + "prepack": "pnpm run clean && pnpm run build" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/riverqueue/river.git", + "directory": "js/driver/mysql" + }, + "contributors": [ + "Brandur Leach", + "Blake Gentry" + ], + "license": "MPL-2.0", + "publishConfig": { + "access": "public", + "provenance": true + }, + "peerDependencies": { + "mysql2": ">=3.0.0", + "riverqueue": "workspace:0.3.1" + }, + "devDependencies": { + "mysql2": "^3.24.4", + "riverqueue": "workspace:0.3.1", + "typescript": "^6.0.3" + }, + "keywords": [ + "river", + "job-queue", + "mysql", + "driver" + ] +} diff --git a/js/driver/mysql/src/codecs.ts b/js/driver/mysql/src/codecs.ts new file mode 100644 index 000000000..d8b980d62 --- /dev/null +++ b/js/driver/mysql/src/codecs.ts @@ -0,0 +1,131 @@ +import { parseJson, stringifyJson, toJsonObject } from "riverqueue"; +import type { JobRow } from "riverqueue"; +import { + decodeAttemptErrors, + decodeJobState, + recordQueueMetadataText, + uniqueBitmaskToStates, +} from "riverqueue/unstable-driver"; +import type { QueueRow } from "riverqueue/unstable-driver"; + +export type Row = Record; + +// Cast at the SQL boundary so caller pool options cannot truncate IDs, JSON +// numbers, or microseconds, or interpret UTC DATETIME values in local time. +export const JOB_COLUMNS = + `CAST(id AS CHAR) AS id, args, attempt, attempted_at, attempted_by, +created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, +scheduled_at, state, tags, unique_key, unique_states`.replace( + /\b(args|attempted_by|errors|metadata|tags|attempted_at|created_at|finalized_at|scheduled_at)\b/g, + "CAST($1 AS CHAR) AS $1" + ); +export const QUEUE_COLUMNS = `name, CAST(created_at AS CHAR) AS created_at, +CAST(metadata AS CHAR) AS metadata, CAST(paused_at AS CHAR) AS paused_at, CAST(updated_at AS CHAR) AS updated_at`; + +export function timestamp( + value: Temporal.Instant | null | undefined +): string | null { + return value == null + ? null + : value + .toString({ fractionalSecondDigits: 6 }) + .replace("T", " ") + .replace(/Z$/, ""); +} + +export function instant(value: unknown): Temporal.Instant { + if (typeof value !== "string") + throw new TypeError("MySQL timestamp is not text"); + return Temporal.Instant.from(`${value.replace(" ", "T")}Z`); +} + +function strings(value: unknown): string[] { + if (value === null) return []; + const array = parseJson(text(value)); + if (array === null) return []; + if (!Array.isArray(array) || array.some((item) => typeof item !== "string")) + throw new TypeError("Expected a JSON array of strings"); + return array as string[]; +} + +export function decodeJob(row: Row): JobRow { + const result = decodeJobPartial(row); + if (result.error) throw result.error; + return result.job; +} + +export function decodeJobPartial(row: Row): { job: JobRow; error?: Error } { + const errors: Error[] = []; + const decode = (field: string, fallback: T, callback: () => T): T => { + try { + return callback(); + } catch (error) { + errors.push( + new Error( + `invalid ${field}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error } + ) + ); + return fallback; + } + }; + const job: JobRow = { + args: decode("args", {}, () => toJsonObject(parseJson(String(row.args)))), + attempt: Math.max(0, Number(row.attempt)), + attemptedAt: row.attempted_at === null ? null : instant(row.attempted_at), + attemptedBy: decode("attempted_by", [], () => strings(row.attempted_by)), + createdAt: instant(row.created_at), + errors: decode("errors", [], () => + row.errors === null ? [] : decodeAttemptErrors(text(row.errors)) + ), + finalizedAt: row.finalized_at === null ? null : instant(row.finalized_at), + id: BigInt(String(row.id)), + kind: String(row.kind), + maxAttempts: Math.max(0, Number(row.max_attempts)), + metadata: decode("metadata", {}, () => + toJsonObject(parseJson(String(row.metadata))) + ), + priority: Number(row.priority), + queue: String(row.queue), + scheduledAt: instant(row.scheduled_at), + state: decodeJobState(String(row.state)), + tags: decode("tags", [], () => strings(row.tags)), + uniqueKey: + row.unique_key === null + ? null + : new Uint8Array(row.unique_key as Uint8Array), + uniqueStates: + row.unique_states === null + ? null + : uniqueBitmaskToStates(Number(row.unique_states)), + }; + return errors.length + ? { + job, + error: new AggregateError( + errors, + `Invalid MySQL job row: ${errors.map((error) => error.message).join("; ")}` + ), + } + : { job }; +} + +export function decodeQueue(row: Row): QueueRow { + const queue = { + createdAt: instant(row.created_at), + metadata: toJsonObject(parseJson(String(row.metadata))), + name: String(row.name), + pausedAt: row.paused_at === null ? null : instant(row.paused_at), + updatedAt: instant(row.updated_at), + }; + recordQueueMetadataText(queue, String(row.metadata)); + return queue; +} + +export { stringifyJson as json }; + +export function text(value: unknown): string { + if (typeof value !== "string") + throw new TypeError("Expected MySQL text column"); + return value; +} diff --git a/js/driver/mysql/src/driver.integration.test.ts b/js/driver/mysql/src/driver.integration.test.ts new file mode 100644 index 000000000..16ed2a327 --- /dev/null +++ b/js/driver/mysql/src/driver.integration.test.ts @@ -0,0 +1,994 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { createPool, type Pool, type RowDataPacket } from "mysql2/promise"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + Client, + TransactionScopeError, + defineJob, + Workers, + parseJson, + isExactJsonNumber, + jsonNumberToBigInt, + stringifyJson, + type ExactJsonNumber, + type JobState, + type JsonObject, +} from "riverqueue"; +import { PilotClient, type PilotDatabase } from "riverqueue/unstable-driver"; +import type { PoolConnection } from "mysql2/promise"; +import type { + JobInsertParams, + JobListParams, + JobCompletionCommand, +} from "riverqueue/unstable-driver"; +import { createMigrator } from "../../../migrate/src/index.js"; +import { decodeAttemptError } from "../../../src/driver-codecs.js"; +import { completionCommand } from "../../../src/runtime/completion-command.js"; +import { MySqlDriver, MySqlRuntime } from "./driver.js"; + +function fixture(name: string) { + return parseJson( + readFileSync( + new URL(`../../../../conformance/testdata/${name}.json`, import.meta.url), + "utf8" + ) + ); +} + +const url = process.env.TEST_MYSQL_URL; +const listing: JobListParams = { + after: null, + ids: [], + kinds: [], + limit: 100, + metadata: null, + priorities: [], + queues: [], + sortDirection: "asc", + sortField: "id", + states: [], + tagsAll: [], + tagsAny: [], +}; +function params(overrides: Partial = {}): JobInsertParams { + return { + args: { message: "hello" }, + encodedArgs: '{"message":"hello"}', + kind: "message", + maxAttempts: 25, + metadata: {}, + priority: 1, + queue: "default", + // Use the claim clock so small client/server clock differences cannot defer a job. + scheduledAt: Temporal.Now.instant(), + state: "available", + tags: [], + uniqueKey: null, + uniqueStates: null, + ...overrides, + }; +} + +describe.skipIf(!url)("MySQL driver", () => { + let pool: Pool; + let schema: string; + let driver: MySqlDriver; + let runtime: MySqlRuntime; + beforeEach(async () => { + pool = createPool({ uri: url!, connectionLimit: 12 }); + schema = `js_${randomUUID().replaceAll("-", "")}`; + driver = new MySqlDriver(pool, { schema }); + runtime = new MySqlRuntime(pool, { schema }); + await createMigrator(driver).migrateUp(); + }); + afterEach(async () => { + await pool.query(`DROP DATABASE IF EXISTS \`${schema}\``); + await pool.end(); + }); + + function pilot(): PilotDatabase { + let database: PilotDatabase | undefined; + class Capture extends PilotClient { + constructor() { + super(driver, { logger: false }, (binding) => { + database = binding; + return {}; + }); + } + } + new Capture(); + if (database === undefined) throw new Error("missing MySQL pilot binding"); + return database; + } + + it("migrates a database with a 64-character name", async () => { + const longSchema = `js_${randomUUID().replaceAll("-", "")}`.padEnd(64, "x"); + const migrator = createMigrator( + new MySqlDriver(pool, { schema: longSchema }) + ); + try { + await migrator.migrateUp(); + expect((await migrator.validate()).ok).toBe(true); + await migrator.migrateDown({ targetVersion: 0 }); + expect(await migrator.existingVersions()).toEqual([]); + } finally { + await pool.query(`DROP DATABASE IF EXISTS \`${longSchema}\``); + } + }); + + it("migrates independent lines alongside the main line", async () => { + const main = createMigrator(driver); + const mainVersions = await main.existingVersions(); + const extra = createMigrator(driver, { + line: "extra", + migrations: [ + { + version: 1, + name: "widget", + upSql: + "CREATE TABLE /* TEMPLATE: schema */widget (id BIGINT PRIMARY KEY)", + downSql: "DROP TABLE /* TEMPLATE: schema */widget", + }, + ], + }); + await extra.migrateUp(); + expect(await extra.existingVersions()).toEqual([1]); + expect(await main.existingVersions()).toEqual(mainVersions); + await extra.migrateDown({ targetVersion: 0 }); + expect(await extra.existingVersions()).toEqual([]); + expect(await main.existingVersions()).toEqual(mainVersions); + }); + + it("pilot transactions preserve rollback errors and caller ownership", async () => { + const database = pilot(), + sentinel = new Error("application rollback"); + await expect( + database.transaction(async (tx) => { + await runtime.jobInsertMany([params()], { tx }); + throw sentinel; + }) + ).rejects.toBe(sentinel); + expect(await runtime.jobList(listing)).toEqual([]); + const tx = await pool.getConnection(); + try { + await tx.beginTransaction(); + await database.transaction( + async (handle) => { + expect(handle).toBe(tx); + await runtime.jobInsertMany([params()], { tx: handle }); + }, + { tx } + ); + await tx.rollback(); + expect(await runtime.jobList(listing)).toEqual([]); + } finally { + tx.release(); + } + }); + + it("pilot connections roll back callbacks that leave a transaction open", async () => { + const database = pilot(); + await expect( + database.connection(async (tx) => { + await tx.beginTransaction(); + await runtime.jobInsertMany([params()], { tx }); + }) + ).rejects.toBeInstanceOf(TransactionScopeError); + expect(await runtime.jobList(listing)).toEqual([]); + expect( + await database.connection( + async (tx) => + (await tx.query("SELECT 1 AS value"))[0][0]?.value + ) + ).toBe(1); + }); + + it("pilot connection cancellation releases the owning pool slot", async () => { + const database = pilot(), + controller = new AbortController(), + reason = new Error("cancel query"); + const pending = database.connection( + async (tx) => { + controller.abort(reason); + await tx.query("SELECT SLEEP(10)"); + }, + { signal: controller.signal } + ); + await expect(pending).rejects.toBe(reason); + expect( + await database.connection( + async (tx) => + (await tx.query("SELECT 1 AS value"))[0][0]?.value + ) + ).toBe(1); + }); + + it("pilot loads exact claims and commits notifications with their transaction", async () => { + const database = pilot(); + const [inserted] = await runtime.jobInsertMany([ + params({ metadata: { label: "é雪" } }), + ]); + expect(inserted).toBeDefined(); + const id = inserted!.job.id; + await database.transaction(async (tx) => { + const result = await database.loadClaimed([id], { tx }); + expect(result.jobs.map((job) => job.id)).toEqual([id]); + expect(result.jobs[0]?.metadata.label).toBe("é雪"); + await expect(database.loadClaimed([id, id], { tx })).rejects.toThrow( + "distinct" + ); + await database.notify("control", ['{"action":"cancel"}'], { tx }); + }); + const [rows] = await pool.query( + `SELECT payload FROM \`${schema}\`.river_notification WHERE topic='river_control'` + ); + expect(rows).toHaveLength(1); + }); + + it("preserves exact queue names", async () => { + const names = ["queue", "queue ", "QUEUE", "cafĂ©", "cafe"]; + for (const name of names) { + expect( + (await runtime.runtimeQueueUpsert(name, Temporal.Now.instant())).name + ).toBe(name); + } + for (const name of names) { + expect((await runtime.queueGet(name))?.name).toBe(name); + } + }); + + it("migrates down and back up", async () => { + const migrator = createMigrator(driver); + expect((await migrator.validate()).ok).toBe(true); + await migrator.migrateDown({ targetVersion: 0 }); + expect(await migrator.existingVersions()).toEqual([]); + await migrator.migrateUp(); + expect((await migrator.validate()).ok).toBe(true); + }); + + it("preserves large IDs, JSON integers, microseconds, and time zones", async () => { + const time = Temporal.Instant.from("2026-10-01T02:03:04.123456Z"); + await pool.query( + `ALTER TABLE \`${schema}\`.river_job AUTO_INCREMENT=9007199254740993` + ); + const first = await runtime.jobInsert( + params({ + encodedArgs: '{"n":9007199254740993}', + args: parseJson('{"n":9007199254740993}') as { n: number }, + createdAt: time, + scheduledAt: time, + }) + ); + const second = await runtime.jobInsert(params()); + expect(first.job.id).toBe(9007199254740993n); + expect(second.job.id).toBe(first.job.id + 1n); + expect(isExactJsonNumber(first.job.args.n)).toBe(true); + expect(first.job.createdAt.equals(time)).toBe(true); + expect(first.job.scheduledAt.equals(time)).toBe(true); + expect( + (await runtime.jobList({ ...listing, ids: [second.job.id] })).map( + (job) => job.id + ) + ).toEqual([second.job.id]); + }); + + it("joins caller transactions and keeps rollback atomic", async () => { + const tx = await pool.getConnection(); + try { + await expect(runtime.jobInsert(params(), { tx })).rejects.toMatchObject({ + code: "transaction_scope", + }); + await tx.beginTransaction(); + const inserted = await runtime.jobInsert(params(), { tx }); + await runtime.notifyInsert(["default"], { tx }); + expect(await runtime.jobGet(inserted.job.id)).toBeNull(); + await tx.rollback(); + expect(await runtime.jobGet(inserted.job.id)).toBeNull(); + const [rows] = await pool.query( + `SELECT * FROM \`${schema}\`.river_notification` + ); + expect(rows).toEqual([]); + } finally { + tx.release(); + } + }); + + it("deduplicates concurrent inserts without changing the existing kind", async () => { + const uniqueKey = new Uint8Array(32).fill(3); + const inserts = await Promise.all( + Array.from({ length: 8 }, () => + runtime.jobInsert(params({ uniqueKey, uniqueStates: ["available"] })) + ) + ); + expect(inserts.filter((item) => item.status === "inserted")).toHaveLength( + 1 + ); + expect(new Set(inserts.map((item) => item.job.id)).size).toBe(1); + const duplicate = await runtime.jobInsert( + params({ kind: "different", uniqueKey, uniqueStates: ["available"] }) + ); + expect(duplicate.status).toBe("duplicate"); + expect(duplicate.job.kind).toBe("message"); + }); + + it("allows repeated batch keys outside the unique states", async () => { + const uniqueKey = new Uint8Array(32).fill(4); + const pending = params({ + uniqueKey, + uniqueStates: ["available"], + state: "pending", + }); + const active = params({ uniqueKey, uniqueStates: ["available"] }); + const inserted = await runtime.jobInsertMany([pending, active, pending]); + expect(inserted.map((item) => item.status)).toEqual([ + "inserted", + "inserted", + "inserted", + ]); + expect(new Set(inserted.map((item) => item.job.id)).size).toBe(3); + await expect( + runtime.jobInsertMany([active, pending, active]) + ).rejects.toThrow("Batch repeats a unique key"); + expect(await runtime.jobList(listing)).toHaveLength(3); + }); + + it("rejects unrelated unique constraints when the incoming key is inactive", async () => { + await pool.query( + `CREATE UNIQUE INDEX custom_kind ON \`${schema}\`.river_job(kind)` + ); + const uniqueKey = new Uint8Array(32).fill(5); + const original = await runtime.jobInsert( + params({ uniqueKey, uniqueStates: ["available"] }) + ); + await expect( + runtime.jobInsert( + params({ uniqueKey, uniqueStates: ["available"], state: "pending" }) + ) + ).rejects.toMatchObject({ + cause: { + message: "Insertion violated an unrelated unique constraint", + cause: { code: "ER_DUP_ENTRY", errno: 1062 }, + }, + }); + expect(await runtime.jobGet(original.job.id)).toEqual(original.job); + expect(await runtime.jobList(listing)).toHaveLength(1); + }); + + it("claims concurrently without duplicate work, and respects paused queues", async () => { + await runtime.runtimeQueueUpsert("paused", Temporal.Now.instant()); + await runtime.queuePause("paused"); + await runtime.jobInsert(params({ queue: "paused" })); + await runtime.jobInsertMany(Array.from({ length: 20 }, () => params())); + const batches = await Promise.all( + Array.from({ length: 4 }, (_, index) => + runtime.jobClaim({ + attemptedBy: `worker${index}`, + kinds: [], + queues: [ + { name: "default", limit: 10 }, + { name: "paused", limit: 1 }, + ], + }) + ) + ); + const ids = batches.flatMap((batch) => batch.jobs.map((job) => job.id)); + // A peer may skip rows while the first claimant holds secondary-index + // locks. The next poll must claim the remainder, with no duplicate IDs. + const remaining = await runtime.jobClaim({ + attemptedBy: "next", + kinds: [], + queues: [{ name: "default", limit: 20 }], + }); + ids.push(...remaining.jobs.map((job) => job.id)); + expect(ids).toHaveLength(20); + expect(new Set(ids).size).toBe(20); + await runtime.queueResume("paused"); + expect( + ( + await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "paused", limit: 1 }], + }) + ).jobs + ).toHaveLength(1); + }); + + it("fences completions and makes a cancellation win over a retry", async () => { + const inserted = await runtime.jobInsert( + params({ metadata: { keep: true, nested: { a: 1 } } }) + ); + const claimed = await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const job = claimed.jobs[0]!; + const command: JobCompletionCommand = { + id: job.id, + attempt: job.attempt, + attemptedBy: "worker", + kind: "retry", + finalizedAt: null, + scheduledAt: Temporal.Now.instant().add({ hours: 1 }), + error: { at: Temporal.Now.instant(), error: "retry", trace: "" }, + metadata: { nested: { b: 2 }, present: null }, + output: null, + outputSet: true, + }; + expect( + (await runtime.jobCompleteMany([{ ...command, attempt: 999 }]))[0]?.status + ).toBe("stale"); + await runtime.jobCancel(inserted.job.id); + const result = (await runtime.jobCompleteMany([command]))[0]!; + expect(result.status).toBe("applied"); + expect(result.job?.state).toBe("cancelled"); + const retried = await runtime.jobRetry(job.id); + expect(retried?.state).toBe("available"); + expect(retried?.metadata).not.toHaveProperty("cancel_attempted_at"); + expect(retried?.metadata.keep).toBe(true); + expect(result.job?.metadata).toMatchObject({ + keep: true, + nested: { b: 2 }, + present: null, + output: null, + }); + expect(result.job?.metadata.nested).toEqual({ b: 2 }); + expect(result.job?.errors).toHaveLength(1); + }); + + it.each([ + [{ value: { a: 1, b: 2 } }, { value: { a: 1 } }, true], + [{ value: { a: 1, b: 2 } }, { value: { a: 2 } }, false], + [{ value: [[1]] }, { value: [1] }, false], + [{ value: [1] }, { value: 1 }, false], + [{ value: [{ a: 1, b: 2 }] }, { value: [{ b: 2 }] }, true], + [{ tenant: { settings: [] } }, { tenant: { settings: {} } }, false], + [{ tenant: { settings: { a: 1 } } }, { tenant: { settings: {} } }, true], + ])( + "preserves nested metadata containment %#", + async (metadata, wanted, matches) => { + await runtime.jobInsert(params({ metadata: { unrelated: true } })); + const inserted = await runtime.jobInsert(params({ metadata })); + const rows = await runtime.jobList({ ...listing, metadata: wanted }); + expect(rows.map((row) => row.id)).toEqual( + matches ? [inserted.job.id] : [] + ); + } + ); + + it("updates queues and jobs, retries and deletes", async () => { + await runtime.runtimeQueueUpsert("default", Temporal.Now.instant()); + expect( + (await runtime.queueUpdate("default", { metadata: { concurrency: 5 } })) + ?.metadata + ).toEqual({ concurrency: 5 }); + expect( + await runtime.queueList({ nameAfter: null, limit: 10 }) + ).toHaveLength(1); + const inserted = await runtime.jobInsert(params()); + await runtime.jobUpdate(inserted.job.id, { + metadata: { nested: { value: true } }, + output: null, + }); + expect((await runtime.jobCancel(inserted.job.id))?.state).toBe("cancelled"); + expect((await runtime.jobRetry(inserted.job.id))?.state).toBe("available"); + expect((await runtime.jobDelete(inserted.job.id)).status).toBe("deleted"); + await runtime.jobInsertMany([params(), params({ kind: "other" })]); + expect( + await runtime.jobDeleteMany({ ...listing, all: false, kinds: ["other"] }) + ).toHaveLength(1); + }); + + it("cleans expired jobs without waiting for locked jobs", async () => { + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "cleaner", + now, + 60_000, + null + ))!; + const [running, locked, ...expired] = await runtime.jobInsertMany( + Array.from({ length: 1002 }, () => params()) + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='completed', finalized_at='1970-01-01' WHERE id>=?`, + [locked!.job.id.toString()] + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='running' WHERE id=?`, + [running!.job.id.toString()] + ); + const tx = await pool.getConnection(); + try { + await tx.beginTransaction(); + for (const id of [running!.job.id, locked!.job.id]) { + await tx.query( + `SELECT id FROM \`${schema}\`.river_job WHERE id=? FOR UPDATE`, + [id.toString()] + ); + } + expect( + await runtime.maintenanceCleanJobs( + leader, + { + cancelledBefore: now, + completedBefore: now, + discardedBefore: now, + limit: 1000, + }, + null, + AbortSignal.timeout(2_000) + ) + ).toBe(1000); + expect(await runtime.jobGet(expired[0]!.job.id)).toBeNull(); + expect((await runtime.jobGet(locked!.job.id))?.state).toBe("completed"); + expect((await runtime.jobGet(running!.job.id))?.state).toBe("running"); + } finally { + await tx.rollback(); + tx.release(); + } + }); + + it("schedules, rescues and cleans under an exact leader term", async () => { + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "leader", + now, + 60_000, + null + ))!; + expect(leader).not.toBeNull(); + expect( + await runtime.maintenanceLeaderAcquire("follower", now, 60_000, null) + ).toBeNull(); + const inserted = await runtime.jobInsert( + params({ state: "scheduled", scheduledAt: now }) + ); + expect( + await runtime.maintenanceSchedule(leader, { + now, + limit: 100, + notificationHorizon: now, + scheduledAtHorizon: now, + allowInsertNotifications: (queues) => queues, + }) + ).toBe(1); + await runtime.jobClaim({ + attemptedBy: "worker", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const horizon = Temporal.Now.instant().add({ hours: 1 }); + expect( + await runtime.maintenanceGetStuck(leader, horizon, 0n, 100) + ).toHaveLength(1); + expect( + await runtime.maintenanceRescue(leader, horizon, [ + { + id: inserted.job.id, + state: "discarded", + finalizedAt: now, + scheduledAt: now, + error: { at: now, attempt: 1, error: "stuck", trace: "" }, + }, + ]) + ).toBe(1); + expect( + (await runtime.jobGet(inserted.job.id))?.metadata["river:rescue_count"] + ).toBe(1); + expect( + await runtime.maintenanceCleanJobs( + leader, + { + limit: 100, + cancelledBefore: horizon, + completedBefore: horizon, + discardedBefore: horizon, + }, + null, + new AbortController().signal + ) + ).toBe(1); + expect(await runtime.maintenanceLeaderResign(leader)).toBe(true); + expect( + await runtime.maintenanceSchedule(leader, { + now, + limit: 100, + notificationHorizon: now, + scheduledAtHorizon: now, + allowInsertNotifications: (queues) => queues, + }) + ).toBe(0); + }); + + it("delivers request_resign and waits for earlier uncommitted notifications", async () => { + const abort = new AbortController(); + const messages: string[] = []; + const ready = Promise.withResolvers(); + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["leadership", "insert"], + abort.signal, + () => ready.resolve(undefined) + )) + messages.push(message.payload); + })(); + await ready.promise; + const first = await pool.getConnection(); + try { + await first.beginTransaction(); + await runtime.notifyInsert(["first"], { tx: first }); + await runtime.notifyInsert(["second"]); + await first.commit(); + await runtime.runtimeRequestLeadershipResignation(); + await expect.poll(() => messages.length).toBe(3); + expect(messages.map((message) => JSON.parse(message))).toEqual([ + { queue: "first" }, + { queue: "second" }, + { action: "request_resign", leader_id: "" }, + ]); + } finally { + first.release(); + abort.abort(); + await listening; + } + }); + + it("interrupts a notification poll blocked on an uncommitted row", async () => { + const abort = new AbortController(); + const ready = Promise.withResolvers(); + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["insert"], + abort.signal, + () => ready.resolve(undefined) + )) + expect(message).toBeUndefined(); + })(); + await ready.promise; + const first = await pool.getConnection(); + try { + await first.beginTransaction(); + await runtime.notifyInsert(["blocked"], { tx: first }); + await expect + .poll(async () => { + const [rows] = await pool.query< + import("mysql2/promise").RowDataPacket[] + >( + "SELECT COUNT(*) AS n FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'", + [schema] + ); + return Number(rows[0]?.n); + }) + .toBeGreaterThan(0); + abort.abort(); + await listening; + expect(await runtime.jobList(listing)).toEqual([]); + } finally { + abort.abort(); + await first.rollback(); + first.release(); + await listening; + } + }); + + it("matches job kinds and queues exactly, including trailing spaces", async () => { + const wanted = (await runtime.jobInsert(params())).job; + await runtime.jobInsert(params({ kind: "message ", queue: "default " })); + expect( + (await runtime.jobList({ ...listing, kinds: ["message"] })).map( + (job) => job.id + ) + ).toEqual([wanted.id]); + expect( + (await runtime.jobList({ ...listing, queues: ["default"] })).map( + (job) => job.id + ) + ).toEqual([wanted.id]); + }); + + it("runs the public worker API to completion", async () => { + const message = defineJob({ kind: "message" }); + const workers = new Workers().add(message, () => undefined); + const client = new Client(driver, { + queues: { default: { maxWorkers: 2 } }, + workers, + completionFlushInterval: { milliseconds: 1 }, + }); + const run = await client.start(); + try { + const inserted = await client.insert(message, {}); + await expect + .poll(async () => (await client.jobs.get(inserted.job.id))?.state, { + timeout: 5000, + }) + .toBe("completed"); + } finally { + await run.stop(); + } + }); + + describe("Go conformance", () => { + if (!url) return; + + const protocol = fixture("protocol_values") as unknown as { + attempt_error: JsonObject; + job_states: { state: JobState; unique_bit: number }[]; + metadata_keys: { + output: string; + rescue_count: string; + unique_nonce: string; + }; + notifications: { name: string; topic: string; payload: JsonObject }[]; + }; + const snoozes = fixture("snooze_counters") as unknown as { + snooze_counters: { + name: string; + metadata: JsonObject; + expected_snoozes: ExactJsonNumber | number; + }[]; + }; + + it.each(protocol.job_states)( + "stores $state and its unique bit", + async ({ state, unique_bit: bit }) => { + const inserted = await runtime.jobInsert( + params({ + uniqueKey: new Uint8Array(32).fill(1), + uniqueStates: [state], + }) + ); + const [encoded] = await pool.query( + `SELECT unique_states FROM \`${schema}\`.river_job WHERE id=?`, + [inserted.job.id.toString()] + ); + expect(encoded[0]?.unique_states).toBe(bit); + const finalized = ["cancelled", "completed", "discarded"].includes( + state + ); + // Seed the Go state/mask independently of the driver's encoder. + await pool.query( + `UPDATE \`${schema}\`.river_job SET state=?, finalized_at=${finalized ? "UTC_TIMESTAMP(6)" : "NULL"}, unique_states=? WHERE id=?`, + [state, bit, inserted.job.id.toString()] + ); + const stored = await runtime.jobGet(inserted.job.id); + expect(stored?.state).toBe(state); + expect(stored?.uniqueStates).toEqual([state]); + expect( + await runtime.jobInsert( + params({ + uniqueKey: inserted.job.uniqueKey, + uniqueStates: ["available", state], + }) + ) + ).toMatchObject({ status: "duplicate", job: { id: inserted.job.id } }); + const [rows] = await pool.query( + `SELECT unique_states FROM \`${schema}\`.river_job WHERE id=?`, + [inserted.job.id.toString()] + ); + expect(rows[0]?.unique_states).toBe(bit); + } + ); + + it.each(snoozes.snooze_counters)( + "persists the $name snooze counter", + async ({ metadata, expected_snoozes: expected }) => { + const inserted = await runtime.jobInsert(params({ metadata })); + const { jobs } = await runtime.jobClaim({ + attemptedBy: "client", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const now = Temporal.Now.instant(); + const command = completionCommand( + jobs[0]!, + "client", + { + status: "succeeded", + outcome: { + type: "snooze", + duration: Temporal.Duration.from({ hours: 1 }), + }, + }, + now, + now, + now, + 1000 + ); + expect((await runtime.jobCompleteMany([command]))[0]?.status).toBe( + "applied" + ); + const stored = (await runtime.jobGet(inserted.job.id))!; + expect(stored.state).toBe("scheduled"); + expect(stored.attempt).toBe(0); + expect( + jsonNumberToBigInt( + stored.metadata.snoozes as ExactJsonNumber | number + ) + ).toBe(jsonNumberToBigInt(expected)); + } + ); + + it.each([ + { initial: 2, expected: 3 }, + { initial: 2.5, expected: 3.5 }, + { initial: null, expected: 1 }, + { initial: "2", expected: 1 }, + ])( + "round trips Go's attempt error and persists rescue metadata from $initial", + async ({ initial, expected }) => { + const keys = protocol.metadata_keys; + const inserted = await runtime.jobInsert( + params({ metadata: { [keys.rescue_count]: initial } }) + ); + const nonce = inserted.job.metadata[keys.unique_nonce]; + expect(nonce).toEqual(expect.any(String)); + expect(nonce).not.toBe(""); + await pool.query( + `UPDATE \`${schema}\`.river_job SET state='running', attempt=3, attempted_at=UTC_TIMESTAMP(6), errors=JSON_ARRAY(CAST(? AS JSON)) WHERE id=?`, + [stringifyJson(protocol.attempt_error), inserted.job.id.toString()] + ); + const error = decodeAttemptError(stringifyJson(protocol.attempt_error)); + expect((await runtime.jobGet(inserted.job.id))?.errors).toEqual([ + error, + ]); + const now = Temporal.Now.instant(); + const leader = (await runtime.maintenanceLeaderAcquire( + "client", + now, + 60_000, + null + ))!; + expect( + await runtime.maintenanceRescue(leader, now.add({ hours: 1 }), [ + { + id: inserted.job.id, + state: "retryable", + scheduledAt: now, + finalizedAt: null, + error, + }, + ]) + ).toBe(1); + await runtime.maintenanceSchedule(leader, { + now, + limit: 1, + scheduledAtHorizon: now, + notificationHorizon: now, + allowInsertNotifications: (queues) => queues, + }); + const { jobs } = await runtime.jobClaim({ + attemptedBy: "client", + kinds: [], + queues: [{ name: "default", limit: 1 }], + }); + const command = completionCommand( + jobs[0]!, + "client", + { + status: "succeeded", + outcome: { type: "complete", output: { ok: true } }, + }, + now, + now, + now, + 1000 + ); + expect((await runtime.jobCompleteMany([command]))[0]?.status).toBe( + "applied" + ); + const stored = (await runtime.jobGet(inserted.job.id))!; + expect(stored.metadata).toMatchObject({ + [keys.output]: { ok: true }, + [keys.rescue_count]: expected, + [keys.unique_nonce]: nonce, + }); + expect(stored.errors).toEqual([error, error]); + } + ); + + it.each(protocol.notifications)( + "emits Go's $name topic and payload", + async ({ name, topic, payload }) => { + // Inputs deliberately use API semantics, not the fixture's wire keys. + switch (name) { + case "cancel": { + const inserted = await runtime.jobInsert( + params({ queue: "priority" }) + ); + await pool.query( + `UPDATE \`${schema}\`.river_job SET id=42, state='running' WHERE id=?`, + [inserted.job.id.toString()] + ); + await runtime.jobCancel(42n); + break; + } + case "insert": + await new Client(driver).insert( + defineJob({ kind: "notify" }), + {}, + { queue: "priority" } + ); + break; + case "metadata_changed": + case "pause": + case "resume": + await runtime.runtimeQueueUpsert( + "priority", + Temporal.Now.instant() + ); + if (name === "metadata_changed") + await runtime.queueUpdate("priority", { + metadata: { owner: "candidate" }, + }); + else { + await runtime.queuePause("priority"); + if (name === "resume") await runtime.queueResume("priority"); + } + break; + case "request_resign": + await runtime.runtimeRequestLeadershipResignation(); + break; + case "resigned": { + const leader = (await runtime.maintenanceLeaderAcquire( + "client-1", + Temporal.Now.instant(), + 60_000, + null + ))!; + expect(await runtime.maintenanceLeaderResign(leader)).toBe(true); + break; + } + default: + throw new Error(`Unhandled Go notification: ${name}`); + } + const [rows] = await pool.query( + `SELECT topic, CAST(payload AS CHAR) AS payload FROM \`${schema}\`.river_notification ORDER BY id DESC LIMIT 1` + ); + expect(rows[0]?.topic).toBe(topic); + expect(parseJson(rows[0]?.payload as string)).toEqual(payload); + } + ); + + it("receives Go's notifications from the MySQL outbox", async () => { + const abort = new AbortController(); + const ready = Promise.withResolvers(); + const received: { topic: string; payload: JsonObject }[] = []; + const listening = (async () => { + for await (const message of runtime.runtimeNotificationSubscribe( + ["control", "insert", "leadership"], + abort.signal, + () => ready.resolve(undefined) + )) { + received.push({ + topic: message.topic, + payload: parseJson(message.payload) as JsonObject, + }); + } + })(); + try { + await ready.promise; + for (const { topic, payload } of protocol.notifications) { + await pool.query( + `INSERT INTO \`${schema}\`.river_notification(topic,payload) VALUES (?,?)`, + [topic, stringifyJson(payload)] + ); + } + await expect + .poll(() => received) + .toEqual( + protocol.notifications.map(({ topic, payload }) => ({ + // The runtime exposes logical topics; the outbox uses Go's prefix. + topic: topic.replace(/^river_/, ""), + payload, + })) + ); + } finally { + abort.abort(); + await listening; + } + }); + }); +}); diff --git a/js/driver/mysql/src/driver.ts b/js/driver/mysql/src/driver.ts new file mode 100644 index 000000000..72f26cf16 --- /dev/null +++ b/js/driver/mysql/src/driver.ts @@ -0,0 +1,1339 @@ +import { Buffer } from "node:buffer"; +import { randomBytes } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { + Pool, + PoolConnection, + ResultSetHeader, + RowDataPacket, +} from "mysql2/promise"; +import { + ConfigurationError, + DatabaseOperationError, + RiverError, + TransactionScopeError, + ValidationError, + isExactJsonNumber, +} from "riverqueue"; +import type { JobRow, JobState, JsonObject, JsonValue } from "riverqueue"; +import { + jobCompletionKey, + jobListKeyset, + jobListKeysetSql, + queueMetadataUpdate, + registerDriver, + uniqueBitmaskFromStates, +} from "riverqueue/unstable-driver"; +import type { + DriverRecord, + FinalizedJobDeleteParams, + PilotDatabase, + DriverInsertResult, + InsertDriverOptions, + JobClaimOptions, + JobClaimParams, + JobClaimResult, + JobCompletionCommand, + JobCompletionResult, + JobDeleteManyParams, + JobDeleteResult, + JobInsertParams, + JobListParams, + JobUpdateParams, + QueueListParams, + QueueRow, + QueueUpdateParams, + RuntimeDriver, + RuntimeJobCleanupParams, + RuntimeJobRescue, + RuntimeLeader, + RuntimeMaintenanceBatch, + RuntimeNotification, + RuntimeScheduleParams, + RuntimeWaitOptions, +} from "riverqueue/unstable-driver"; +import { + decodeJob, + decodeJobPartial, + decodeQueue, + instant, + JOB_COLUMNS, + json, + QUEUE_COLUMNS, + timestamp, + text, +} from "./codecs.js"; +import type { Row } from "./codecs.js"; + +/** MySQL database containing River's tables; defaults to the pool's database. */ +export interface MySqlDriverOptions { + readonly schema?: string; +} +type Options = InsertDriverOptions & RuntimeWaitOptions; +const NONCE = "river:unique_nonce"; + +/** + * MySQL 8.0.17+ backend using a caller-owned mysql2 pool. Pass an open + * connection as `{ tx }` to join an application transaction. River never + * closes the pool or commits a caller's transaction. + */ +export class MySqlDriver { + declare readonly "~river"?: { + readonly capability: "runtime"; + readonly transaction: PoolConnection; + }; + constructor(pool: Pool, options: MySqlDriverOptions = {}) { + const runtime = new MySqlRuntime(pool, options); + registerDriver(this, runtime.driverRecord()); + } +} + +/** @internal */ +export class MySqlRuntime implements RuntimeDriver { + readonly backend = "mysql"; + readonly #pool: Pool; + readonly #prefix: string; + readonly #schema: string | undefined; + + constructor(pool: Pool, options: MySqlDriverOptions = {}) { + if (typeof pool.getConnection !== "function") + throw new ConfigurationError( + "MySqlDriver requires a mysql2 promise Pool" + ); + if ( + options.schema !== undefined && + !/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(options.schema) + ) + throw new ConfigurationError("Invalid MySQL schema name"); + this.#pool = pool; + this.#schema = options.schema; + this.#prefix = options.schema === undefined ? "" : `\`${options.schema}\`.`; + } + + driverRecord(): DriverRecord { + return { + backend: this.backend, + capability: "runtime", + operations: this, + database: this.#pilotDatabase(), + migration: { backend: "mysql", pool: this.#pool, schema: this.#schema }, + }; + } + + #pilotDatabase(): PilotDatabase { + return { + backend: this.backend, + schema: this.#schema ?? null, + connection: (callback, options = {}) => + this.#pilotConnection(callback, options.signal), + transaction: (callback, options = {}) => + this.#transaction( + options, + async (tx) => { + const result = await callback(tx); + options.signal?.throwIfAborted(); + return result; + }, + true + ), + deleteFinalizedJobs: (params, options = {}) => + this.#deleteFinalizedJobs(params, options), + loadClaimed: (ids, options) => + this.#transaction(options, async (tx) => { + if (new Set(ids).size !== ids.length) + throw new ValidationError("claimed job IDs must be distinct"); + const jobs: JobRow[] = [], + decodeErrors = new Map(); + for (const id of ids) { + const row = await this.#get(tx, id, true); + if (row === null) + throw new ValidationError(`claimed job ${id} does not exist`); + const result = decodeJobPartial(row); + jobs.push(result.job); + if (result.error) decodeErrors.set(id, result.error); + } + return { jobs, decodeErrors }; + }), + notify: (topic, payloads, options) => + this.#transaction(options, async (tx) => { + for (const payload of payloads) + await this.#notify(tx, topic, payload); + }), + }; + } + + async #pilotConnection( + callback: (tx: PoolConnection) => T | PromiseLike, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted(); + const acquiring = this.#pool.getConnection(); + let abort: (() => void) | undefined; + let tx: PoolConnection; + try { + tx = + signal === undefined + ? await acquiring + : await Promise.race([ + acquiring, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + }), + ]); + } catch (error) { + void acquiring.then( + (late) => late.release(), + () => undefined + ); + throw error; + } finally { + if (abort !== undefined) signal?.removeEventListener("abort", abort); + } + const closeTransaction = async (): Promise => { + const [status] = await tx.query("DO 0"); + if ((status.serverStatus & 1) === 0) return false; + await tx.rollback(); + return true; + }; + const aborted = { value: false }; + try { + signal?.throwIfAborted(); + const operation = async () => { + let result: T; + try { + result = await callback(tx); + } catch (error) { + if (!aborted.value) { + try { + await closeTransaction(); + } catch { + tx.destroy(); + } + } + throw error; + } + if (aborted.value) throw signal?.reason; + try { + if (await closeTransaction()) + throw new TransactionScopeError( + "nested", + "a pilot connection callback left a transaction open; River rolled it back" + ); + } catch (error) { + if (!(error instanceof TransactionScopeError)) tx.destroy(); + throw error; + } + return result; + }; + if (signal === undefined) return await operation(); + return await Promise.race([ + new Promise((_, reject) => { + abort = () => { + aborted.value = true; + tx.destroy(); + reject(signal.reason); + }; + signal.addEventListener("abort", abort, { once: true }); + }), + operation(), + ]); + } finally { + if (abort !== undefined) signal?.removeEventListener("abort", abort); + tx.release(); + } + } + + async #deleteFinalizedJobs( + params: FinalizedJobDeleteParams, + options: { readonly tx?: PoolConnection } + ): Promise { + return this.#transaction(options, async (tx) => { + const clauses: string[] = [], + values: unknown[] = []; + for (const [state, before] of [ + ["cancelled", params.cancelledBefore], + ["completed", params.completedBefore], + ["discarded", params.discardedBefore], + ] as const) { + if (before != null) { + clauses.push("(state=? AND finalized_at "?").join(",")})` + ); + values.push(...params.queuesIncluded); + } + if (params.queuesExcluded?.length) { + filters.push( + `queue NOT IN (${params.queuesExcluded.map(() => "?").join(",")})` + ); + values.push(...params.queuesExcluded); + } + const rows = await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE ${filters.join(" AND ")} ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED`, + [...values, params.limit] + ); + if (rows.length === 0) return 0; + return this.#write( + tx, + `DELETE FROM {schema}river_job WHERE id IN (${rows.map(() => "?").join(",")})`, + rows.map((row) => row.id) + ); + }); + } + + async operationScope( + tx: PoolConnection | undefined, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#transaction(tx === undefined ? {} : { tx }, callback); + } + + async #transaction( + options: Options, + callback: (tx: PoolConnection) => Promise, + preserveCallbackError = false + ): Promise { + options.signal?.throwIfAborted(); + if (options.tx !== undefined) { + if ( + typeof options.tx.execute !== "function" || + typeof options.tx.release !== "function" + ) + throw new ConfigurationError( + "Expected a mysql2 pool connection transaction" + ); + const [status] = await options.tx.query("DO 0"); + if ((status.serverStatus & 1) === 0) + throw new TransactionScopeError( + "no_transaction", + "MySQL connection has no open transaction" + ); + return callback(options.tx); + } + const acquiring = this.#pool.getConnection(); + let tx: PoolConnection; + if (options.signal) { + const signal = options.signal; + let abort!: () => void; + try { + tx = await Promise.race([ + acquiring, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + }), + ]); + } catch (error) { + void acquiring.then( + (late) => late.release(), + () => undefined + ); + throw error; + } finally { + signal.removeEventListener("abort", abort); + } + } else tx = await acquiring; + let abort: (() => void) | undefined; + const callbackFailed = { value: false }; + try { + options.signal?.throwIfAborted(); + const operation = async () => { + await tx.query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED"); + await tx.beginTransaction(); + try { + const result = await callback(tx).catch((error: unknown) => { + callbackFailed.value = true; + throw error; + }); + await tx.commit(); + return result; + } catch (error) { + try { + await tx.rollback(); + } catch { + tx.destroy(); + } + throw error; + } + }; + if (!options.signal) return await operation(); + const signal = options.signal; + return await Promise.race([ + new Promise((_, reject) => { + abort = () => { + // A pending locking read must not delay shutdown or return to + // the pool. Only River-owned connections are cancelled this way. + tx.destroy(); + reject(signal.reason); + }; + signal.addEventListener("abort", abort, { once: true }); + }), + operation(), + ]); + } catch (cause) { + options.signal?.throwIfAborted(); + if ( + cause instanceof RiverError || + (preserveCallbackError && callbackFailed.value) + ) + throw cause; + const code = (cause as { code?: string }).code ?? ""; + throw new DatabaseOperationError("MySQL operation failed", { + backend: "mysql", + operation: "transaction", + cause, + retryable: + /DEADLOCK|LOCK_WAIT_TIMEOUT|ECONN|EPIPE|ETIMEDOUT|PROTOCOL_CONNECTION_LOST/.test( + code + ), + }); + } finally { + if (abort) options.signal?.removeEventListener("abort", abort); + tx.release(); + } + } + + async #rows( + tx: PoolConnection, + sql: string, + values: readonly unknown[] = [] + ): Promise { + const [rows] = await tx.execute( + sql.replaceAll("{schema}", this.#prefix), + values.map(binding) + ); + return rows; + } + async #write( + tx: PoolConnection, + sql: string, + values: readonly unknown[] = [] + ): Promise { + const [result] = await tx.execute( + sql.replaceAll("{schema}", this.#prefix), + values.map(binding) + ); + return result.affectedRows; + } + async #get( + tx: PoolConnection, + id: bigint, + lock = false + ): Promise { + return ( + ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE id=?${lock ? " FOR UPDATE" : ""}`, + [id] + ) + )[0] ?? null + ); + } + + async jobInsert( + params: JobInsertParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const metadata = { + ...params.metadata, + [NONCE]: randomBytes(8).toString("hex"), + }; + const insertSql = `INSERT INTO {schema}river_job (args,created_at,kind,max_attempts,metadata,priority,queue,scheduled_at,state,tags,unique_key,unique_states) + VALUES (CAST(? AS JSON),COALESCE(?,UTC_TIMESTAMP(6)),?,?,CAST(? AS JSON),?,?,COALESCE(?,UTC_TIMESTAMP(6)),?,CAST(? AS JSON),?,?) + `; + const values = [ + params.encodedArgs, + timestamp(params.createdAt), + params.kind, + Math.min(params.maxAttempts, 2_147_483_647), + json(metadata), + params.priority, + params.queue, + timestamp(params.scheduledAt), + params.state, + json(params.tags), + params.uniqueKey === null ? null : Buffer.from(params.uniqueKey), + params.uniqueStates === null + ? null + : Number.parseInt(uniqueBitmaskFromStates(params.uniqueStates), 2), + ]; + await this.#write( + tx, + `${insertSql} ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id)`, + values + ); + const row = ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE id=LAST_INSERT_ID() FOR UPDATE` + ) + )[0]; + if (!row) throw new Error("MySQL insert returned no row"); + const job = decodeJob(row); + const duplicate = job.metadata[NONCE] !== metadata[NONCE]; + if ( + duplicate && + (params.uniqueKey === null || + !params.uniqueStates?.includes(params.state) || + job.uniqueKey === null || + !Buffer.from(params.uniqueKey).equals(job.uniqueKey) || + !job.uniqueStates?.includes(job.state)) + ) { + // MySQL cannot target one index in ON DUPLICATE KEY. Preserve the + // native constraint error for callers that own additional indexes. + try { + await this.#write(tx, insertSql, values); + } catch (cause) { + throw new Error("Insertion violated an unrelated unique constraint", { + cause, + }); + } + throw new Error("Insertion violated an unrelated unique constraint"); + } + return { job, status: duplicate ? "duplicate" : "inserted" }; + }); + } + async jobInsertMany( + params: readonly JobInsertParams[], + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const keys = new Set(); + const results = []; + for (const item of params) { + if ( + item.uniqueKey !== null && + item.uniqueStates?.includes(item.state) + ) { + const key = Buffer.from(item.uniqueKey).toString("hex"); + if (keys.has(key)) + throw new ConfigurationError("Batch repeats a unique key"); + keys.add(key); + } + results.push(await this.jobInsert(item, { tx })); + } + return results; + }); + } + async jobGet(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id); + return row ? decodeJob(row) : null; + }); + } + async jobCancel(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return null; + const job = decodeJobPartial(row).job; + if ( + ["cancelled", "completed", "discarded"].includes(job.state) || + job.finalizedAt !== null + ) + return job; + const now = Temporal.Now.instant(); + await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=CASE WHEN state='running' THEN finalized_at ELSE ? END, + metadata=JSON_SET(metadata,'$.cancel_attempted_at',?), state=CASE WHEN state='running' THEN state ELSE 'cancelled' END WHERE id=?`, + [timestamp(now), now.toString(), id] + ); + if (job.state === "running") + await this.#notify( + tx, + "control", + `{"action":"cancel","job_id":${id},"queue":${JSON.stringify(job.queue)}}` + ); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async jobDelete(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return { status: "not_found" }; + const job = decodeJobPartial(row).job; + if (job.state === "running") return { job, status: "running" }; + await this.#write(tx, "DELETE FROM {schema}river_job WHERE id=?", [id]); + return { job, status: "deleted" }; + }); + } + async jobDeleteMany( + params: JobDeleteManyParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const { where, values } = filters(params); + if (!params.all && where.length === 0) + throw new ConfigurationError("Deleting all jobs requires all=true"); + where.push("state != 'running'"); + const rows = await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE ${where.join(" AND ")} ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED`, + [...values, params.limit] + ); + for (const row of rows) + await this.#write(tx, "DELETE FROM {schema}river_job WHERE id=?", [ + row.id, + ]); + return rows.map((row) => decodeJobPartial(row).job); + }); + } + async jobList( + params: JobListParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const { where, values } = filters(params); + if (params.metadata !== null) { + where.push(metadataContains(params.metadata, "metadata", values)); + } + if (params.tagsAll.length) { + where.push("JSON_CONTAINS(tags,CAST(? AS JSON))"); + values.push(json(params.tagsAll)); + } + if (params.tagsAny.length) { + where.push("JSON_OVERLAPS(tags,CAST(? AS JSON))"); + values.push(json(params.tagsAny)); + } + const keyset = jobListKeyset(params); + const order = jobListKeysetSql(keyset, (value) => { + values.push(typeof value === "bigint" ? value : timestamp(value)); + return "?"; + }); + if (order.after) where.push(order.after); + const orderBy = order.orderBy.replace( + /(\w+) (ASC|DESC) NULLS (LAST|FIRST)/g, + (_, field: string, direction: string, nulls: string) => + `${field} IS NULL ${nulls === "LAST" ? "ASC" : "DESC"}, ${field} ${direction}` + ); + return ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE ${where.join(" AND ") || "true"} ORDER BY ${orderBy} LIMIT ?`, + [...values, params.limit] + ) + ).map(decodeJob); + }); + } + async jobRetry(id: bigint, options: Options = {}): Promise { + return this.#transaction(options, async (tx) => { + const row = await this.#get(tx, id, true); + if (!row) return null; + const job = decodeJobPartial(row).job; + const now = Temporal.Now.instant(); + if ( + job.state === "running" || + (job.state === "available" && + Temporal.Instant.compare(job.scheduledAt, now) < 0) + ) + return job; + await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=NULL, metadata=JSON_REMOVE(metadata,'$.cancel_attempted_at'), max_attempts=CASE WHEN attempt=max_attempts THEN max_attempts+1 ELSE max_attempts END, scheduled_at=?, state='available' WHERE id=?`, + [timestamp(now), id] + ); + await this.notifyInsert([job.queue], { tx }); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async #metadata( + tx: PoolConnection, + id: bigint, + metadata: JsonObject + ): Promise { + const entries = Object.entries(metadata); + if (!entries.length) return; + const values = entries.flatMap(([key, value]) => [ + `$.${JSON.stringify(key)}`, + json(value), + ]); + await this.#write( + tx, + `UPDATE {schema}river_job SET metadata=JSON_SET(metadata,${entries.map(() => "?,CAST(? AS JSON)").join(",")}) WHERE id=?`, + [...values, id] + ); + } + async jobUpdate( + id: bigint, + params: JobUpdateParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + if (!(await this.#get(tx, id, true))) return null; + await this.#metadata(tx, id, { + ...params.metadata, + ...(params.output === undefined ? {} : { output: params.output }), + }); + return decodeJobPartial(required(await this.#get(tx, id, true))).job; + }); + } + async jobClaim( + params: JobClaimParams, + options: JobClaimOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const jobs: JobRow[] = []; + const decodeErrors = new Map(); + const now = timestamp(Temporal.Now.instant()); + for (const queue of params.queues) { + if (!queue.limit) continue; + const kindSql = params.kinds.length + ? ` AND CAST(kind AS BINARY) IN (${params.kinds.map(() => "?").join(",")})` + : ""; + // Keep an indexable equality so claims do not scan and lock other queues. + const rows = await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE queue=? AND queue=CAST(? AS BINARY) AND state='available' AND scheduled_at<=? + AND NOT EXISTS(SELECT 1 FROM {schema}river_queue q WHERE q.name=CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL)${kindSql} + ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED`, + [queue.name, queue.name, now, ...params.kinds, queue.limit] + ); + for (const row of rows) { + await this.#write( + tx, + `UPDATE {schema}river_job SET attempted_at=?, attempted_by=JSON_ARRAY_APPEND(CASE WHEN JSON_TYPE(attempted_by) != 'ARRAY' OR attempted_by IS NULL THEN JSON_ARRAY() WHEN JSON_LENGTH(attempted_by)>=100 THEN JSON_EXTRACT(attempted_by,'$[last-98 to last]') ELSE attempted_by END,'$',?), attempt=attempt+1,state='running' WHERE id=?`, + [now, params.attemptedBy, row.id] + ); + const result = decodeJobPartial( + required(await this.#get(tx, BigInt(String(row.id)), true)) + ); + jobs.push(result.job); + if (result.error) decodeErrors.set(result.job.id, result.error); + } + } + return { jobs, decodeErrors }; + }); + } + async jobCompleteMany( + commands: readonly JobCompletionCommand[], + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const results: JobCompletionResult[] = []; + for (const command of commands) { + const raw = await this.#get(tx, command.id, true); + const job = raw ? decodeJobPartial(raw).job : null; + const key = jobCompletionKey(command); + if ( + !job || + job.attempt !== command.attempt || + job.attemptedBy.at(-1) !== command.attemptedBy + ) { + results.push({ job, key, status: "stale" }); + continue; + } + let state: JobState = + command.kind === "complete" + ? "completed" + : command.kind === "cancel" + ? "cancelled" + : command.kind === "discard" + ? "discarded" + : command.kind === "interrupt" || command.available + ? "available" + : command.kind === "retry" + ? "retryable" + : "scheduled"; + const cancelled = + ["available", "retryable", "scheduled"].includes(state) && + Object.hasOwn(job.metadata, "cancel_attempted_at"); + let finalizedAt = command.finalizedAt; + let scheduledAt = command.scheduledAt ?? job.scheduledAt; + let attempt = + command.kind === "snooze" || command.kind === "interrupt" + ? Math.max(0, command.attempt - 1) + : command.attempt; + if (cancelled) { + state = "cancelled"; + finalizedAt = Temporal.Now.instant(); + scheduledAt = job.scheduledAt; + attempt = job.attempt; + } + if (job.state === "running") { + if ( + ["cancelled", "completed", "discarded"].includes(state) !== + (finalizedAt !== null) + ) + throw new ConfigurationError("Invalid completion timestamp"); + await this.#write( + tx, + `UPDATE {schema}river_job SET attempt=?,finalized_at=?,scheduled_at=?,state=?, + errors=CASE WHEN ? THEN JSON_ARRAY_APPEND(CASE WHEN errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(errors)='ARRAY' THEN errors ELSE JSON_ARRAY(errors) END,'$',CAST(? AS JSON)) ELSE errors END WHERE id=?`, + [ + attempt, + timestamp(finalizedAt), + timestamp(scheduledAt), + state, + command.error !== null, + command.error === null + ? "null" + : json({ + ...command.error, + at: command.error.at.toString(), + attempt: command.attempt, + }), + command.id, + ] + ); + } + await this.#metadata(tx, command.id, { + ...command.metadata, + ...(command.outputSet ? { output: command.output } : {}), + }); + results.push({ + job: decodeJobPartial(required(await this.#get(tx, command.id, true))) + .job, + key, + status: job.state === "running" ? "applied" : "stale", + }); + } + return results; + }); + } + + async #notify( + tx: PoolConnection, + topic: RuntimeNotification["topic"], + payload: string + ): Promise { + await this.#write( + tx, + "INSERT INTO {schema}river_notification(topic,payload) VALUES (?,?)", + [`river_${topic}`, payload] + ); + } + async notifyInsert( + queues: readonly string[], + options: Options = {} + ): Promise { + await this.#transaction(options, async (tx) => { + for (const queue of new Set(queues)) + await this.#notify(tx, "insert", json({ queue })); + }); + } + async jobGetCancelRequested( + ids: readonly bigint[], + options: RuntimeWaitOptions = {} + ): Promise { + if (!ids.length) return []; + return this.#transaction(options, async (tx) => + ( + await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE id IN (${ids.map(() => "?").join(",")}) AND state='running' AND JSON_CONTAINS_PATH(metadata,'one','$.cancel_attempted_at')`, + ids + ) + ).map((row) => BigInt(String(row.id))) + ); + } + async runtimeRequestLeadershipResignation( + options: Options = {} + ): Promise { + await this.#transaction(options, (tx) => + this.#notify( + tx, + "leadership", + json({ action: "request_resign", leader_id: "" }) + ) + ); + } + async *runtimeNotificationSubscribe( + topics: readonly RuntimeNotification["topic"][], + signal: AbortSignal, + ready: () => void + ): AsyncIterable { + try { + let after = await this.#transaction({ signal }, async (tx) => { + const rows = await this.#rows( + tx, + "SELECT CAST(id AS CHAR) AS id FROM {schema}river_notification ORDER BY id FOR UPDATE" + ); + return BigInt(text(rows.at(-1)?.id ?? "0")); + }); + ready(); + while (!signal.aborted) { + const rows = await this.#transaction({ signal }, (tx) => + this.#rows( + tx, + "SELECT CAST(id AS CHAR) AS id,topic,payload FROM {schema}river_notification WHERE id>? ORDER BY id LIMIT 256 FOR SHARE", + [after] + ) + ); + for (const row of rows) { + after = BigInt(String(row.id)); + const topic = String(row.topic).replace( + /^river_/, + "" + ) as RuntimeNotification["topic"]; + if (topics.includes(topic)) + yield { topic, payload: String(row.payload) }; + } + try { + await sleep(25, undefined, { signal }); + } catch (error) { + if (!(error instanceof Error && error.name === "AbortError")) + throw error; + } + } + } catch (error) { + if (!signal.aborted) throw error; + } + } + + async #queue( + tx: PoolConnection, + name: string, + lock = false + ): Promise { + const row = ( + await this.#rows( + tx, + `SELECT ${QUEUE_COLUMNS} FROM {schema}river_queue WHERE name=CAST(? AS BINARY)${lock ? " FOR UPDATE" : ""}`, + [name] + ) + )[0]; + return row ? decodeQueue(row) : null; + } + async queueGet( + name: string, + options: Options = {} + ): Promise { + return this.#transaction(options, (tx) => this.#queue(tx, name)); + } + async queueList( + params: QueueListParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => + ( + await this.#rows( + tx, + `SELECT ${QUEUE_COLUMNS} FROM {schema}river_queue WHERE (? IS NULL OR name>?) ORDER BY name LIMIT ?`, + [params.nameAfter, params.nameAfter, params.limit] + ) + ).map(decodeQueue) + ); + } + async queuePause( + name: string, + options: Options = {} + ): Promise { + return this.#setPaused(name, true, options); + } + async queueResume( + name: string, + options: Options = {} + ): Promise { + return this.#setPaused(name, false, options); + } + async #setPaused( + name: string, + paused: boolean, + options: Options + ): Promise { + return this.#transaction(options, async (tx) => { + const condition = paused ? "paused_at IS NULL" : "paused_at IS NOT NULL"; + await this.#write( + tx, + `UPDATE {schema}river_queue SET updated_at=CASE WHEN ${condition} THEN UTC_TIMESTAMP(6) ELSE updated_at END, paused_at=${paused ? "COALESCE(paused_at,UTC_TIMESTAMP(6))" : "NULL"} WHERE (?='*' OR name=CAST(? AS BINARY))`, + [name, name] + ); + await this.#notify( + tx, + "control", + json({ action: paused ? "pause" : "resume", queue: name }) + ); + return this.#queue(tx, name); + }); + } + async queueUpdate( + name: string, + params: QueueUpdateParams, + options: Options = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const before = await this.#queue(tx, name, true); + if (!before) return null; + const updated = queueMetadataUpdate(name, params); + if (updated) { + await this.#write( + tx, + "UPDATE {schema}river_queue SET metadata=CAST(? AS JSON),updated_at=UTC_TIMESTAMP(6) WHERE name=CAST(? AS BINARY)", + [updated.text, name] + ); + await this.#notify(tx, "control", updated.notification); + } + return this.#queue(tx, name); + }); + } + async runtimeQueueUpsert( + name: string, + now: Temporal.Instant, + options: RuntimeWaitOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + await this.#write( + tx, + "INSERT INTO {schema}river_queue(name,created_at,updated_at) VALUES (?,?,?) ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at)", + [name, timestamp(now), timestamp(now)] + ); + return required(await this.#queue(tx, name, true)); + }); + } + + async maintenanceLeaderAcquire( + leaderId: string, + now: Temporal.Instant, + ttlMs: number, + held: RuntimeLeader | null, + options: RuntimeWaitOptions = {} + ): Promise { + return this.#transaction(options, async (tx) => { + const expiresAt = now.add({ milliseconds: ttlMs }); + if (held) { + const updated = await this.#write( + tx, + "UPDATE {schema}river_leader SET expires_at=? WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at>?", + [ + timestamp(expiresAt), + held.leaderId, + timestamp(held.electedAt), + timestamp(now), + ] + ); + return updated ? { ...held, expiresAt } : null; + } + await this.#write( + tx, + "DELETE FROM {schema}river_leader WHERE name='default' AND expires_at { + return this.#transaction({}, async (tx) => { + const changed = await this.#write( + tx, + "DELETE FROM {schema}river_leader WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=?", + [leader.leaderId, timestamp(leader.electedAt)] + ); + if (changed) + await this.#notify( + tx, + "leadership", + json({ action: "resigned", leader_id: leader.leaderId }) + ); + return changed > 0; + }); + } + async #fenced( + leader: RuntimeLeader, + options: Options, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#transaction(options, async (tx) => { + const rows = await this.#rows( + tx, + "SELECT leader_id FROM {schema}river_leader WHERE name='default' AND leader_id=CAST(? AS BINARY) AND elected_at=? AND expires_at>UTC_TIMESTAMP(6) FOR SHARE", + [leader.leaderId, timestamp(leader.electedAt)] + ); + if (!rows.length) return null; + return callback(tx); + }); + } + async maintenanceTransaction( + leader: RuntimeLeader, + batch: RuntimeMaintenanceBatch, + callback: (tx: PoolConnection) => Promise + ): Promise { + return this.#fenced(leader, { signal: batch.signal }, async (tx) => { + const result = await callback(tx); + batch.signal.throwIfAborted(); + return result; + }); + } + async maintenanceSchedule( + leader: RuntimeLeader, + params: RuntimeScheduleParams, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced( + leader, + batch ? { signal: batch.signal } : {}, + async (tx) => { + const rows = await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE state IN ('retryable','scheduled') AND scheduled_at<=? ORDER BY priority,scheduled_at,id LIMIT ? FOR UPDATE SKIP LOCKED`, + [timestamp(params.scheduledAtHorizon), params.limit] + ); + const queues: string[] = []; + for (const row of rows) { + const job = decodeJobPartial(row).job; + // A savepoint contains only the conflicting state change, leaving the + // batch usable after MySQL's duplicate-key error. + await tx.query("SAVEPOINT river_schedule"); + try { + await this.#write( + tx, + "UPDATE {schema}river_job SET state='available' WHERE id=?", + [job.id] + ); + if ( + Temporal.Instant.compare( + job.scheduledAt, + params.notificationHorizon + ) <= 0 + ) + queues.push(job.queue); + } catch (error) { + if ((error as { code?: string }).code !== "ER_DUP_ENTRY") + throw error; + await tx.query("ROLLBACK TO SAVEPOINT river_schedule"); + await this.#write( + tx, + "UPDATE {schema}river_job SET state='discarded',finalized_at=?,metadata=JSON_SET(metadata,'$.unique_key_conflict','scheduler_discarded') WHERE id=?", + [timestamp(params.now), job.id] + ); + } finally { + await tx.query("RELEASE SAVEPOINT river_schedule"); + } + } + await this.notifyInsert(params.allowInsertNotifications(queues), { + tx, + }); + return rows.length; + } + )) ?? 0 + ); + } + async maintenanceGetStuck( + leader: RuntimeLeader, + attemptedBefore: Temporal.Instant, + afterId: bigint, + limit: number, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced( + leader, + batch ? { signal: batch.signal } : {}, + async (tx) => + ( + await this.#rows( + tx, + `SELECT ${JOB_COLUMNS} FROM {schema}river_job WHERE state='running' AND attempted_at? ORDER BY id LIMIT ?`, + [timestamp(attemptedBefore), afterId, limit] + ) + ).map((row) => decodeJobPartial(row).job) + )) ?? [] + ); + } + async maintenanceRescue( + leader: RuntimeLeader, + attemptedBefore: Temporal.Instant, + jobs: readonly RuntimeJobRescue[], + options: Options = {} + ): Promise { + return ( + (await this.#fenced(leader, options, async (tx) => { + let count = 0; + for (const job of jobs) { + const raw = await this.#get(tx, job.id, true); + if (!raw) continue; + const before = decodeJobPartial(raw).job; + if ( + before.state !== "running" || + before.attemptedAt === null || + Temporal.Instant.compare(before.attemptedAt, attemptedBefore) >= 0 + ) + continue; + count += await this.#write( + tx, + `UPDATE {schema}river_job SET finalized_at=?,scheduled_at=?,state=?,metadata=JSON_SET(metadata,'$."river:rescue_count"',COALESCE(CASE WHEN JSON_TYPE(JSON_EXTRACT(metadata,'$."river:rescue_count"')) IN ('INTEGER','DOUBLE') THEN JSON_EXTRACT(metadata,'$."river:rescue_count"') ELSE NULL END,0)+1),errors=JSON_ARRAY_APPEND(COALESCE(errors,JSON_ARRAY()),'$',CAST(? AS JSON)) WHERE id=?`, + [ + timestamp(job.finalizedAt), + timestamp(job.scheduledAt), + job.state, + json({ ...job.error, at: job.error.at.toString() }), + job.id, + ] + ); + } + return count; + })) ?? 0 + ); + } + async maintenanceCleanJobs( + leader: RuntimeLeader, + params: RuntimeJobCleanupParams, + _timeoutMs: number | null, + signal: AbortSignal + ): Promise { + return ( + (await this.#fenced(leader, { signal }, async (tx) => { + const rows = await this.#rows( + tx, + `SELECT CAST(id AS CHAR) AS id FROM {schema}river_job WHERE ((state='cancelled' AND finalized_at "?").join(",")})` : ""} ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED`, + [ + timestamp(params.cancelledBefore), + timestamp(params.completedBefore), + timestamp(params.discardedBefore), + ...(params.queuesExcluded ?? []), + params.limit, + ] + ); + if (!rows.length) return 0; + return this.#write( + tx, + `DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM {schema}river_job WHERE id IN (${rows.map(() => "?").join(",")})`, + rows.map((row) => row.id) + ); + })) ?? 0 + ); + } + async maintenanceCleanQueues( + leader: RuntimeLeader, + updatedBefore: Temporal.Instant, + limit: number, + batch?: RuntimeMaintenanceBatch + ): Promise { + return ( + (await this.#fenced(leader, batch ? { signal: batch.signal } : {}, (tx) => + this.#write( + tx, + "DELETE FROM {schema}river_queue WHERE updated_at { + return ( + (await this.#fenced(leader, batch ? { signal: batch.signal } : {}, (tx) => + this.#write( + tx, + "DELETE FROM {schema}river_notification WHERE created_at +): { where: string[]; values: unknown[] } { + const where: string[] = []; + const values: unknown[] = []; + for (const [column, items] of [ + ["id", params.ids], + ["CAST(kind AS BINARY)", params.kinds], + ["priority", params.priorities], + ["CAST(queue AS BINARY)", params.queues], + ["state", params.states], + ] as const) { + if (items.length) { + where.push(`${column} IN (${items.map(() => "?").join(",")})`); + values.push(...items); + } + } + return { where, values }; +} + +function binding(value: unknown): string | number | boolean | Buffer | null { + if (typeof value === "bigint") return value.toString(); + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + Buffer.isBuffer(value) + ) + return value; + throw new TypeError("Invalid MySQL SQL parameter"); +} + +// MySQL JSON_CONTAINS flattens nested arrays and accepts scalar-in-array +// matches. Preserve Postgres containment boundaries by testing each element. +// Scalar subqueries avoid MySQL 8 semijoin rewrites losing nested correlations. +function metadataContains( + wanted: JsonValue, + value: string, + values: unknown[], + depth = 0 +): string { + const alias = `metadata_${depth}`; + if ( + wanted !== null && + typeof wanted === "object" && + !isExactJsonNumber(wanted) + ) { + const array = Array.isArray(wanted); + const children = array + ? wanted.map((item) => { + const predicate = metadataContains( + item, + `${alias}.value`, + values, + depth + 1 + ); + return `(SELECT 1 FROM JSON_TABLE(${value}, '$[*]' COLUMNS(value JSON PATH '$')) AS ${alias} WHERE ${predicate} LIMIT 1) IS NOT NULL`; + }) + : Object.entries(wanted).map(([key, item]) => { + values.push(`$.${JSON.stringify(key)}`); + const predicate = metadataContains( + item, + `${alias}.value`, + values, + depth + 1 + ); + return `(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT(${value},?), '$' COLUMNS(value JSON PATH '$')) AS ${alias} WHERE ${predicate} LIMIT 1) IS NOT NULL`; + }); + return `(JSON_TYPE(${value})='${array ? "ARRAY" : "OBJECT"}' AND ${children.join(" AND ") || "true"})`; + } + if (wanted === null) + return `(${value} IS NULL OR JSON_TYPE(${value})='NULL')`; + values.push(json(wanted)); + return `(JSON_TYPE(${value}) NOT IN ('ARRAY','OBJECT') AND JSON_CONTAINS(${value},CAST(? AS JSON)))`; +} + +function required(value: T | null | undefined): T { + if (value == null) throw new Error("MySQL row disappeared while locked"); + return value; +} diff --git a/js/driver/mysql/src/index.ts b/js/driver/mysql/src/index.ts new file mode 100644 index 000000000..4c11778b0 --- /dev/null +++ b/js/driver/mysql/src/index.ts @@ -0,0 +1,11 @@ +import type { PoolConnection } from "mysql2/promise"; + +export { MySqlDriver } from "./driver.js"; +export type { MySqlDriverOptions } from "./driver.js"; + +declare module "riverqueue" { + interface RiverTransactionRegistry { + /** A mysql2 connection with an open transaction. */ + "@riverqueue/driver-mysql": PoolConnection; + } +} diff --git a/js/driver/mysql/tsconfig.json b/js/driver/mysql/tsconfig.json new file mode 100644 index 000000000..fc6b18f33 --- /dev/null +++ b/js/driver/mysql/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "stripInternal": true + }, + "exclude": ["src/**/*.integration.test.ts", "src/**/*.test.ts"], + "include": ["src"] +} diff --git a/js/etc/riverqueue.api.md b/js/etc/riverqueue.api.md index 43114308e..7e2d9e7b3 100644 --- a/js/etc/riverqueue.api.md +++ b/js/etc/riverqueue.api.md @@ -2657,6 +2657,11 @@ interface RiverTransactionRegistry { "@riverqueue/driver-knex": Knex.Transaction; } +interface RiverTransactionRegistry { + /** A mysql2 connection with an open transaction. */ + "@riverqueue/driver-mysql": PoolConnection; +} + interface RiverTransactionRegistry { /** node-postgres clients (`pg.Client` or a pool's `PoolClient`). */ "@riverqueue/driver-pg": ClientBase; diff --git a/js/etc/riverqueue.unstable-driver.api.md b/js/etc/riverqueue.unstable-driver.api.md index 9830babf3..fbcaa4d0a 100644 --- a/js/etc/riverqueue.unstable-driver.api.md +++ b/js/etc/riverqueue.unstable-driver.api.md @@ -212,6 +212,11 @@ export declare function driverMigrationTarget( * a `node:sqlite` database. */ export type DriverMigrationTarget = + | { + readonly backend: "mysql"; + readonly pool: object; + readonly schema: string | undefined; + } | { readonly client: object; readonly schema: string | undefined; diff --git a/js/migrate/etc/migrate.api.md b/js/migrate/etc/migrate.api.md index cc56ac21e..279e8d97a 100644 --- a/js/migrate/etc/migrate.api.md +++ b/js/migrate/etc/migrate.api.md @@ -135,7 +135,7 @@ export declare const MIGRATION_LINE_MAIN = "main"; ```ts /** Database backend that a migration bundle targets. */ -export type MigrationBackend = "postgres" | "sqlite"; +export type MigrationBackend = "mysql" | "postgres" | "sqlite"; ``` ## `MigrationDirection` @@ -162,7 +162,10 @@ export declare class MigrationError extends RiverError<"migration"> { ```ts /** A database connection to migrate, given without a River driver. */ export type MigrationTarget = - PgClientMigrationTarget | PgPoolMigrationTarget | SqliteMigrationTarget; + | MySqlMigrationTarget + | PgClientMigrationTarget + | PgPoolMigrationTarget + | SqliteMigrationTarget; ``` ## `Migrator` @@ -239,6 +242,36 @@ export interface MigratorOptions { export type MigratorSource = ClientDriver | MigrationTarget; ``` +## `MySqlMigrationConnection` + +```ts +/** The mysql2 promise connection methods used by the migrator. */ +export interface MySqlMigrationConnection { + query(sql: string, values?: unknown[]): Promise<[unknown, unknown]>; + release(): void; +} +``` + +## `MySqlMigrationPool` + +```ts +/** A caller-owned mysql2 promise pool. */ +export interface MySqlMigrationPool { + getConnection(): Promise; +} +``` + +## `MySqlMigrationTarget` + +```ts +/** Migrate MySQL through a caller-owned mysql2 pool. */ +export interface MySqlMigrationTarget { + readonly backend: "mysql"; + readonly pool: MySqlMigrationPool; + readonly schema?: string | undefined; +} +``` + ## `PgClientMigrationTarget` ```ts diff --git a/js/migrate/migrations/manifest.json b/js/migrate/migrations/manifest.json index ef9f6f0af..f431e710e 100644 --- a/js/migrate/migrations/manifest.json +++ b/js/migrate/migrations/manifest.json @@ -1,5 +1,71 @@ { "backends": { + "mysql": [ + { + "file": "001_create_river_migration.down.sql", + "sha256": "660d6b11eeb88589d3ce2053be165cda8df4c1d0172fbc1e8c3daa614e3f5b93" + }, + { + "file": "001_create_river_migration.up.sql", + "sha256": "b0d3147cf330526af7f3478b336f3f9f929bb9f217db08da12ca821e6d5568fd" + }, + { + "file": "002_initial_schema.down.sql", + "sha256": "93d74166cb838b13ae42447181afb2c2987dcb16ae33422de612daecb0163e79" + }, + { + "file": "002_initial_schema.up.sql", + "sha256": "2c5ab6645cf016155ad7fd5c46929ddc8bf76006cb6e7b85601bdadcf945f504" + }, + { + "file": "003_river_job_tags_non_null.down.sql", + "sha256": "dbfb0192d6254bdcf2a12c70c8b4ad44dfd7ebaa273b3160c0e76b9208f0a1fe" + }, + { + "file": "003_river_job_tags_non_null.up.sql", + "sha256": "e8a8459edd8e6110e564be74c03f646b430e2f3e31fef31eeb5610ed3b10660f" + }, + { + "file": "004_pending_and_more.down.sql", + "sha256": "f4325dd580c3725d7eb068262d56319f779b556ceddfb53771c51a855ca2440f" + }, + { + "file": "004_pending_and_more.up.sql", + "sha256": "c4642feef2bddc16ca0c5dcdf84942abb219ed40b1427b571673e8f5360bbb53" + }, + { + "file": "005_migration_unique_client.down.sql", + "sha256": "bd06f54a7fd083e4962355d67186fcd09f0f8de6392d18f406003fba0732018f" + }, + { + "file": "005_migration_unique_client.up.sql", + "sha256": "09e8dff3080a05bfc10ff260bb96508cb0e9639a3f9dc86505c50ef6f931dcc3" + }, + { + "file": "006_bulk_unique.down.sql", + "sha256": "5c86b3bd5e32a1f8462b6e4f7ba8aa4fcb4c5be37f4c7d3c1faeb9602ca410fe" + }, + { + "file": "006_bulk_unique.up.sql", + "sha256": "f8625bc2fbff65b1b77fb416bc95889981256d890ce8d157bb1227064bff5228" + }, + { + "file": "007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql", + "sha256": "d25dbb01d5c4385f40581fe9eb54f26aa71709cb7f10050803c5e1fc7a8bf46c" + }, + { + "file": "007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql", + "sha256": "25cc59f31682454bc497962a3a98cf29f5760e3341e2bfa37fcef108a86d9b96" + }, + { + "file": "008_job_id_autoincrement.down.sql", + "sha256": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa" + }, + { + "file": "008_job_id_autoincrement.up.sql", + "sha256": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa" + } + ], "postgres": [ { "file": "001_create_river_migration.down.sql", @@ -135,6 +201,7 @@ }, "format": 1, "sources": { + "mysql": "riverdriver/rivermysql/migration/main", "postgres": "riverdriver/riverpgxv5/migration/main", "sqlite": "riverdriver/riversqlite/migration/main" } diff --git a/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql b/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/js/migrate/migrations/mysql/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql b/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..8a76b2634 --- /dev/null +++ b/js/migrate/migrations/mysql/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/js/migrate/migrations/mysql/main/002_initial_schema.down.sql b/js/migrate/migrations/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/js/migrate/migrations/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/js/migrate/migrations/mysql/main/002_initial_schema.up.sql b/js/migrate/migrations/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..f841bf89a --- /dev/null +++ b/js/migrate/migrations/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL has no LISTEN/NOTIFY transport, so river_job_notify is omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/js/migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql b/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/js/migrate/migrations/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql b/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..b544e5000 --- /dev/null +++ b/js/migrate/migrations/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. States use VARCHAR with an +-- explicit constraint, so changing them does not require a table rebuild. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql b/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..5906e5567 --- /dev/null +++ b/js/migrate/migrations/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql b/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..b1e83547d --- /dev/null +++ b/js/migrate/migrations/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql b/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/js/migrate/migrations/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql b/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/js/migrate/migrations/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..1f368209e --- /dev/null +++ b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..82ec71552 --- /dev/null +++ b/js/migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/js/migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/js/migrate/src/bundle.ts b/js/migrate/src/bundle.ts index 11c2f1e43..07234fbe7 100644 --- a/js/migrate/src/bundle.ts +++ b/js/migrate/src/bundle.ts @@ -4,7 +4,7 @@ import { readFileSync } from "node:fs"; import { MigrationError } from "riverqueue"; /** Database backend that a migration bundle targets. */ -export type MigrationBackend = "postgres" | "sqlite"; +export type MigrationBackend = "mysql" | "postgres" | "sqlite"; /** One River migration version with SQL for both directions. */ export interface Migration { @@ -42,7 +42,7 @@ const MIGRATION_FILE_RE = export function loadMigrations( backend: MigrationBackend ): readonly Migration[] { - if (!["postgres", "sqlite"].includes(backend)) { + if (!["mysql", "postgres", "sqlite"].includes(backend)) { throw new MigrationError( `unsupported migration backend: ${JSON.stringify(backend)}`, { backend, operation: "load" } diff --git a/js/migrate/src/index.ts b/js/migrate/src/index.ts index a27bd2e70..820e3efb9 100644 --- a/js/migrate/src/index.ts +++ b/js/migrate/src/index.ts @@ -18,6 +18,7 @@ export type { Migrator, MigratorOptions, MigratorSource, + MySqlMigrationTarget, PgClientMigrationTarget, PgPoolMigrationTarget, SqliteMigrationTarget, @@ -30,3 +31,5 @@ export type { PgMigrationPoolClient, PgMigrationQueryResult, } from "./postgres.js"; + +export type { MySqlMigrationConnection, MySqlMigrationPool } from "./mysql.js"; diff --git a/js/migrate/src/migrator.ts b/js/migrate/src/migrator.ts index a15397064..6d6ae9e52 100644 --- a/js/migrate/src/migrator.ts +++ b/js/migrate/src/migrator.ts @@ -21,6 +21,7 @@ import { SqliteMigrationStorage, type SqliteMigrationRunner, } from "./sqlite.js"; +import { MySqlMigrationStorage, type MySqlMigrationPool } from "./mysql.js"; import type { MigrationStorage } from "./storage.js"; export { MIGRATION_LINE_MAIN }; @@ -137,7 +138,17 @@ export type MigratorSource = ClientDriver | MigrationTarget; /** A database connection to migrate, given without a River driver. */ export type MigrationTarget = - PgClientMigrationTarget | PgPoolMigrationTarget | SqliteMigrationTarget; + | MySqlMigrationTarget + | PgClientMigrationTarget + | PgPoolMigrationTarget + | SqliteMigrationTarget; + +/** Migrate MySQL through a caller-owned mysql2 pool. */ +export interface MySqlMigrationTarget { + readonly backend: "mysql"; + readonly pool: MySqlMigrationPool; + readonly schema?: string | undefined; +} /** Migrate Postgres through one dedicated connection. */ export interface PgClientMigrationTarget { @@ -216,12 +227,16 @@ export function createMigrator( ): Migrator { const target = resolveTarget(source); const storage: MigrationStorage = - "database" in target - ? new SqliteMigrationStorage(target.database, target.run) - : new PgMigrationStorage( - "pool" in target ? { pool: target.pool } : { client: target.client }, - target.schema - ); + "backend" in target + ? new MySqlMigrationStorage(target.pool, target.schema) + : "database" in target + ? new SqliteMigrationStorage(target.database, target.run) + : new PgMigrationStorage( + "pool" in target + ? { pool: target.pool } + : { client: target.client }, + target.schema + ); const line = options.line ?? MIGRATION_LINE_MAIN; validateLine(storage.backend, line); const migrations = @@ -420,6 +435,7 @@ function copyMigrations( function resolveTarget( source: MigratorSource ): + | MySqlMigrationTarget | PgClientMigrationTarget | PgPoolMigrationTarget | (SqliteMigrationTarget & { readonly run?: SqliteMigrationRunner }) { @@ -437,6 +453,7 @@ function resolveTarget( }; } if (typeof source === "object" && (source as unknown) !== null) { + if ("backend" in source) return source; if ("database" in source) return { database: source.database }; if ("pool" in source) { return { pool: source.pool, schema: source.schema }; diff --git a/js/migrate/src/mysql.ts b/js/migrate/src/mysql.ts new file mode 100644 index 000000000..09acb3f95 --- /dev/null +++ b/js/migrate/src/mysql.ts @@ -0,0 +1,199 @@ +import { MigrationError } from "riverqueue"; +import { isInTargetState, requireLineStorage, versionRecord } from "./plan.js"; +import type { + MigrationSession, + MigrationStep, + MigrationStorage, +} from "./storage.js"; + +/** The mysql2 promise connection methods used by the migrator. */ +export interface MySqlMigrationConnection { + query(sql: string, values?: unknown[]): Promise<[unknown, unknown]>; + release(): void; +} +/** A caller-owned mysql2 promise pool. */ +export interface MySqlMigrationPool { + getConnection(): Promise; +} + +/** MySQL DDL commits implicitly; the named lock spans all migration steps. */ +export class MySqlMigrationStorage implements MigrationStorage { + readonly backend = "mysql"; + readonly #pool: MySqlMigrationPool; + readonly #schema: string | undefined; + readonly #prefix: string; + constructor(pool: MySqlMigrationPool, schema: string | undefined) { + if (schema !== undefined && !/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(schema)) + throw new MigrationError("Invalid MySQL schema name", { + backend: "mysql", + operation: "configure", + }); + this.#pool = pool; + this.#schema = schema; + this.#prefix = schema === undefined ? "" : `\`${schema}\`.`; + } + renderSql(sql: string): string { + return sql.replaceAll("/* TEMPLATE: schema */", this.#prefix); + } + async readVersions(line: string): Promise { + const connection = await this.#pool.getConnection(); + try { + return await this.#versions(connection, line); + } finally { + connection.release(); + } + } + async #versions( + connection: MySqlMigrationConnection, + line: string + ): Promise { + const rows = await query( + connection, + "SELECT column_name AS column_name FROM information_schema.columns WHERE table_schema=COALESCE(?,DATABASE()) AND table_name='river_migration'", + [this.#schema ?? null] + ); + const tableExists = rows.length > 0; + const hasLineColumn = rows.some((row) => row.column_name === "line"); + requireLineStorage("mysql", line, { tableExists, hasLineColumn }); + if (!tableExists) return []; + return ( + await query( + connection, + `SELECT version FROM ${this.#prefix}river_migration${hasLineColumn ? " WHERE line=?" : ""} ORDER BY version`, + hasLineColumn ? [line] : [] + ) + ).map((row) => Number(row.version)); + } + async withSession( + run: (session: MigrationSession) => Promise + ): Promise { + const connection = await this.#pool.getConnection(); + let locked = false; + try { + const result = await query( + connection, + "SELECT GET_LOCK(CONCAT('river_migrate:',LEFT(SHA2(COALESCE(?,DATABASE()),256),48)),0) AS acquired", + [this.#schema ?? null] + ); + if (Number(result[0]?.acquired) !== 1) + throw new Error("Another migrator holds the database lock"); + locked = true; + return await run({ + readVersions: (line) => this.#versions(connection, line), + apply: (step) => this.#apply(connection, step), + }); + } catch (cause) { + if (cause instanceof MigrationError) throw cause; + throw new MigrationError("MySQL migration failed", { + backend: "mysql", + operation: "migrate", + cause, + }); + } finally { + try { + if (locked) + await connection.query( + "SELECT RELEASE_LOCK(CONCAT('river_migrate:',LEFT(SHA2(COALESCE(?,DATABASE()),256),48)))", + [this.#schema ?? null] + ); + } finally { + connection.release(); + } + } + } + async #apply( + connection: MySqlMigrationConnection, + step: MigrationStep + ): Promise { + const { direction, line, migration } = step; + if ( + isInTargetState( + direction, + await this.#versions(connection, line), + migration.version + ) + ) + return false; + if (line === "main" && direction === "down" && migration.version <= 5) { + const rows = await query( + connection, + "SELECT column_name AS column_name FROM information_schema.columns WHERE table_schema=COALESCE(?,DATABASE()) AND table_name='river_migration' AND column_name='line'", + [this.#schema ?? null] + ); + if ( + rows.length && + ( + await query( + connection, + `SELECT 1 FROM ${this.#prefix}river_migration WHERE line<>'main' LIMIT 1` + ) + ).length + ) + throw new Error("Migrate other lines down before the main line"); + } + if (direction === "up" && this.#schema !== undefined) + await connection.query( + `CREATE DATABASE IF NOT EXISTS \`${this.#schema}\`` + ); + for (const sql of statements(step.sql)) await connection.query(sql); + // Record only successful DDL. MySQL cannot roll back partially applied DDL. + const record = versionRecord(direction, line, migration.version); + switch (record.kind) { + case "none": + break; + case "insert": + await connection.query( + `INSERT INTO ${this.#prefix}river_migration(line,version) VALUES (?,?)`, + [record.line, record.version] + ); + break; + case "insert_without_line": + await connection.query( + `INSERT INTO ${this.#prefix}river_migration(version) VALUES (?)`, + [record.version] + ); + break; + case "delete": + await connection.query( + `DELETE FROM ${this.#prefix}river_migration WHERE line=? AND version=?`, + [record.line, record.version] + ); + break; + case "delete_without_line": + await connection.query( + `DELETE FROM ${this.#prefix}river_migration WHERE version=?`, + [record.version] + ); + break; + } + return true; + } +} + +async function query( + connection: MySqlMigrationConnection, + sql: string, + values: unknown[] = [] +): Promise[]> { + const [rows] = await connection.query(sql, values); + return rows as Record[]; +} + +/** Split scripts without requiring MULTI_STATEMENTS on application pools. */ +function statements(sql: string): string[] { + const tokens = + sql.match( + /'(?:''|\\.|[^'\\])*'|"(?:""|\\.|[^"\\])*"|`(?:``|[^`])*`|--[^\n]*|#[^\n]*|\/\*[\s\S]*?\*\/|;|[\s\S]/g + ) ?? []; + const result: string[] = []; + let buffer = ""; + for (const token of tokens) { + if (/^(--|#|\/\*)/.test(token)) buffer += " "; + else if (token === ";") { + if (buffer.trim()) result.push(buffer.trim()); + buffer = ""; + } else buffer += token; + } + if (buffer.trim()) result.push(buffer.trim()); + return result; +} diff --git a/js/package.json b/js/package.json index ff582da4f..4cac4fcf2 100644 --- a/js/package.json +++ b/js/package.json @@ -100,6 +100,7 @@ "knex": "^3.1.0", "knip": "^6.42.1", "license-checker-rseidelsohn": "^5.0.1", + "mysql2": "^3.24.4", "pg": "^8.23.1", "pino": "^10.4.0", "prettier": "^3.9.10", diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index f582195b9..40528f70b 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -48,13 +48,16 @@ importers: version: 4.10.2 knex: specifier: ^3.1.0 - version: 3.3.0(pg@8.23.1) + version: 3.3.0(mysql2@3.24.4(@types/node@26.6.5))(pg@8.23.1) knip: specifier: ^6.42.1 version: 6.42.1 license-checker-rseidelsohn: specifier: ^5.0.1 version: 5.0.1 + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.6.5) pg: specifier: ^8.23.1 version: 8.23.1 @@ -103,6 +106,9 @@ importers: '@riverqueue/migrate': specifier: workspace:0.3.1 version: link:../migrate + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.6.5) pg: specifier: ^8.23.1 version: 8.23.1 @@ -124,7 +130,7 @@ importers: devDependencies: knex: specifier: ^3.1.0 - version: 3.3.0(pg@8.23.1) + version: 3.3.0(mysql2@3.24.4(@types/node@26.6.5))(pg@8.23.1) pg: specifier: ^8.23.1 version: 8.23.1 @@ -135,6 +141,18 @@ importers: specifier: ^6.0.3 version: 6.0.3 + driver/mysql: + devDependencies: + mysql2: + specifier: 3.24.4 + version: 3.24.4(@types/node@26.6.5) + riverqueue: + specifier: workspace:0.3.1 + version: link:../.. + typescript: + specifier: ^6.0.3 + version: 6.0.3 + driver/pg: dependencies: postgres-array: @@ -708,56 +726,48 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@oxc-parser/binding-linux-arm64-musl@0.154.0': resolution: {integrity: sha512-uiwA0V3q8llWA7/8r+a9BBjk4j5YMDmi6wfOsyQfGOJMQSLHEiZ3HI8ulea766ykUa9wJFWk9XWEl4uJrZLQqw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@oxc-parser/binding-linux-ppc64-gnu@0.154.0': resolution: {integrity: sha512-LICfN0Xs4Zdko+bJy6eOAEQojDuIR26suF2JtOsMaXjgbCrpRSH7/NCvJhY13YvYBx12V6hnwrL8n0lk0rbNJQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@oxc-parser/binding-linux-riscv64-gnu@0.154.0': resolution: {integrity: sha512-gbbSYxLF8ssRbQ2qQPy+2EhGNwNH1bAoRdkk6cnWaKHqmCikpSIPtpXjFbCCtjTi0uojqOM3ghFQwQPbDQsU0Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] - libc: [glibc] '@oxc-parser/binding-linux-riscv64-musl@0.154.0': resolution: {integrity: sha512-nSnRYYDLCcRVjSQzF3g6rKqCNYwo5o+aQgnn8BBXfe+ofCgMGlGqkfRyvXm6ptZvoxNkkWtLlqM8W7Xc3j6ZnQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] - libc: [musl] '@oxc-parser/binding-linux-s390x-gnu@0.154.0': resolution: {integrity: sha512-BxBQoJKtzRQgZEKQIt6CDUm8RGKC5T8Qo4576yDa5/xgc10X3fBDItMbRabzLtbUBfTqKLl+8AVr/H6CF7W22g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@oxc-parser/binding-linux-x64-gnu@0.154.0': resolution: {integrity: sha512-/whkust2vYy+GjRUVL5cWaJmEu6jkdFfo5mkxgckyeOX61eMwo4iBoiwzeJZeMWAdzb/XOtQyLApHhFht3sOWA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@oxc-parser/binding-linux-x64-musl@0.154.0': resolution: {integrity: sha512-EXURTyW9riw4B/kZnnE+58emAtw6+3VEszQfXUZkRqGaKIcmWDYkZtKB+xY/blHvuZ+Qy/fDclc31nmUCgtkSA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@oxc-parser/binding-openharmony-arm64@0.154.0': resolution: {integrity: sha512-ZRFooyH3qsR6vuR4jFGlG3XM36490SvvXtNaUahBK4zHpooMdTHNaOMtGGJI0EW38m5hi4rOco9kw7ZwCIp28w==} @@ -825,49 +835,41 @@ packages: resolution: {integrity: sha512-vDT3KHgzYp47gmtNOqL2VNhCyl5Zv643eyxm//A68J8DeUGXrvD1pZFiaT4jSfe+RInfnn1R2yVHye4enx6RnA==} cpu: [arm64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-arm64-musl@11.24.2': resolution: {integrity: sha512-+kMlQvbzfyEYtu5FcjE4p+ttBLpKW4d/AsAsuE69BxV6V4twZJeIQZFfD8gh/wqglY0MkPSezWXQH0jBV13MUw==} cpu: [arm64] os: [linux] - libc: [musl] '@oxc-resolver/binding-linux-ppc64-gnu@11.24.2': resolution: {integrity: sha512-shjfMhmZ3gq9fv/w7bi3PnZlgOPG+2QAOFf0BJF0EgBSIGZ6PMLN2zbGEblTUYB/NKVDRyYhE2ff3dJ1QqNPkA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-riscv64-gnu@11.24.2': resolution: {integrity: sha512-zGelwFR5oRo+b69k8Lrzun86DyUHzfKN6cnjbR9l7Z7NIRznOE/2ZvPa1IUKqAL2PzAXOdwkfVqNvO1H2RlpAw==} cpu: [riscv64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-riscv64-musl@11.24.2': resolution: {integrity: sha512-qxZ1SWCXJY0eyhAlP6Lmo9F2Nrtx7EkYj9oCgL8apDPCwXwCEDA2U697bbT81JIc2IrVjxO4KX6WU2N+oN9Z4w==} cpu: [riscv64] os: [linux] - libc: [musl] '@oxc-resolver/binding-linux-s390x-gnu@11.24.2': resolution: {integrity: sha512-sGCecF3cx2DFlH4t/z7ApnOnXqN48p5p5mlHDEnHTAukQa2P+qMVE4CwyWE9W+q/m3QJ7kKfGrIjax31f44oFQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-x64-gnu@11.24.2': resolution: {integrity: sha512-k/VlMMcSzMlahb3/fENM4rTlsJ0s3fFROA0KXPBmKggqmTSaE383sl8F3KCOXPLmVsYfW6hCitMhXCEtNeZxxg==} cpu: [x64] os: [linux] - libc: [glibc] '@oxc-resolver/binding-linux-x64-musl@11.24.2': resolution: {integrity: sha512-8hbnZyNi97b/8wapYaIF9+t9GmZKBW2vunaOc3h9HGJptH7b7XpvZqOTBSm/MpTjr7H497BlgOaSfLUdhmy2bw==} cpu: [x64] os: [linux] - libc: [musl] '@oxc-resolver/binding-openharmony-arm64@11.24.2': resolution: {integrity: sha512-MvyGik3a6pVgZ0t/kWlbmFxFLmXQJwgLsY2eYFHLpy0wGwRbfzeIGgDwQ3kXqE30z+kSXennRkCrT7TUvkptNg==} @@ -1074,42 +1076,36 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.2.14': resolution: {integrity: sha512-BJ2CL19M6uuWesMgnqJzgxoI+N+eV7b0A9O9vLV42y+gitC41fNa8KElau1tHVzuLWX5GzEuxMPhAO0tQXQMkg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.2.14': resolution: {integrity: sha512-4+Kl8daPV8NDHaD7mfCcc1pRM+Qg7ab8DyMm5FG5WoZgv4h2i3KFN1CybYBNbQqmuYm7ftJLvmMI1eHYJ22S/Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.2.14': resolution: {integrity: sha512-V/h8ePq1fKDrmtFMz6xcCN3DKvuvMRrwwD1q0PkBeeo4DpK64biaKvqWGVxqCeYKqeRk7gE7/iKPxLCn08O97A==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.2.14': resolution: {integrity: sha512-WxalXzgrLVpKC93xfpaJ3gCXUWzeCr+FRCbDVD2mUAy4L6Vo3lNVz0epUEUpxPWJkLNYUwM4OSIikv36OJ9mLg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.2.14': resolution: {integrity: sha512-3G3tnAmTpdk9jO3GQtyHXqeFD2SlecfTws/Y3M2mmj9nMBwwuLT4kur8sZKeu7ENUHSpVO3cIsm5A52LEWWwlg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.2.14': resolution: {integrity: sha512-qyr5XiNGxm2ytx17vvtcJ9nXn6v1gENETi503bJx3OGwZ1tSonj7BosxbMa/xeAfSl5iS9X6KjEWYXLu6v1L7Q==} @@ -2187,28 +2183,24 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.33.0: resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.33.0: resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.33.0: resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.33.0: resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} @@ -4764,7 +4756,7 @@ snapshots: dependencies: '@keyv/serialize': 1.1.1 - knex@3.3.0(pg@8.23.1): + knex@3.3.0(mysql2@3.24.4(@types/node@26.6.5))(pg@8.23.1): dependencies: colorette: 2.0.19 commander: 10.0.1 @@ -4781,6 +4773,7 @@ snapshots: tarn: 3.1.2 tildify: 2.0.0 optionalDependencies: + mysql2: 3.24.4(@types/node@26.6.5) pg: 8.23.1 transitivePeerDependencies: - supports-color diff --git a/js/scripts/api-reports.mjs b/js/scripts/api-reports.mjs index 4ec7cbc8f..1e01daad2 100644 --- a/js/scripts/api-reports.mjs +++ b/js/scripts/api-reports.mjs @@ -81,6 +81,11 @@ const reports = [ name: "@riverqueue/driver-knex", output: "driver/knex/etc/driver-knex.api.md", }, + { + entry: "driver/mysql/dist/index.d.ts", + name: "@riverqueue/driver-mysql", + output: "driver/mysql/etc/driver-mysql.api.md", + }, { entry: "driver/pg/dist/index.d.ts", name: "@riverqueue/driver-pg", diff --git a/js/scripts/check-licenses.mjs b/js/scripts/check-licenses.mjs index 0fb188dec..64f613f98 100644 --- a/js/scripts/check-licenses.mjs +++ b/js/scripts/check-licenses.mjs @@ -10,6 +10,7 @@ const packageDirectories = [ ".", "migrate", "driver/knex", + "driver/mysql", "driver/pg", "driver/prisma", "driver/sqlite", diff --git a/js/scripts/check-packages.mjs b/js/scripts/check-packages.mjs index 6bf8e2ec0..253ea1e7b 100644 --- a/js/scripts/check-packages.mjs +++ b/js/scripts/check-packages.mjs @@ -38,6 +38,7 @@ const packageSpecs = [ { directory: ".", name: "riverqueue" }, { directory: "migrate", name: "@riverqueue/migrate" }, { directory: "driver/knex", name: "@riverqueue/driver-knex" }, + { directory: "driver/mysql", name: "@riverqueue/driver-mysql" }, { directory: "driver/pg", name: "@riverqueue/driver-pg" }, { directory: "driver/prisma", name: "@riverqueue/driver-prisma" }, { directory: "driver/sqlite", name: "@riverqueue/driver-sqlite" }, @@ -374,6 +375,11 @@ async function verifyConsumers(archiveByPackageName) { .filter(([name]) => name !== "@riverqueue/cli") .map(([name, archivePath]) => [name, `file:${archivePath}`]) ); + // Consumers supply the drivers' runtime peers when automatic peer installs + // are disabled to exercise JavaScript usage without type packages. + for (const name of ["@prisma/client", "knex", "mysql2", "pg"]) { + dependencies[name] = rootPackageJson.devDependencies[name]; + } await writeFile( join(consumerDirectory, "package.json"), `${JSON.stringify( @@ -389,8 +395,10 @@ async function verifyConsumers(archiveByPackageName) { )}\n` ); // Install libraries before the CLI, which owns a polyfill for its executable. - // Libraries must not install type packages or a Temporal implementation. - await npmInstall(consumerDirectory); + // Verify JavaScript consumers can run without type packages or a Temporal + // implementation. mysql2 requires @types/node as a peer, so omit peer-only + // dependencies until the TypeScript consumer checks below. + await npmInstall(consumerDirectory, ["--omit=peer"]); await assert.rejects( stat(join(consumerDirectory, "node_modules", "@js-temporal/polyfill")), { code: "ENOENT" }, @@ -402,6 +410,7 @@ async function verifyConsumers(archiveByPackageName) { ` import * as river from "riverqueue"; import * as knexDriver from "@riverqueue/driver-knex"; +import * as mysql from "@riverqueue/driver-mysql"; import * as pg from "@riverqueue/driver-pg"; import * as prisma from "@riverqueue/driver-prisma"; import * as sqlite from "@riverqueue/driver-sqlite"; @@ -409,7 +418,7 @@ import * as migrate from "@riverqueue/migrate"; import * as testing from "@riverqueue/test"; import * as workerThreads from "@riverqueue/worker-threads"; -for (const packageNamespace of [river, knexDriver, pg, prisma, sqlite, migrate, testing, workerThreads]) { +for (const packageNamespace of [river, knexDriver, mysql, pg, prisma, sqlite, migrate, testing, workerThreads]) { if (Object.keys(packageNamespace).length === 0) throw new Error("empty package namespace"); } @@ -435,7 +444,7 @@ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed) || packageJson.dependencies["@riverqueue/cli"] = `file:${archiveByPackageName.get("@riverqueue/cli")}`; await writeFile(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`); - await npmInstall(consumerDirectory); + await npmInstall(consumerDirectory, ["--omit=peer"]); for (const dependency of ["@types/node", "@types/pg"]) { await assert.rejects( stat(join(consumerDirectory, "node_modules", dependency)), @@ -445,7 +454,7 @@ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed) || } // Library consumers still explicitly choose and preload their implementation. - await installTemporalSetup(consumerDirectory); + await installTemporalSetup(consumerDirectory, ["--omit=peer"]); await run(process.execPath, [join(consumerDirectory, "import.mjs")], { temporalDirectory: consumerDirectory, }); @@ -498,6 +507,7 @@ const packageNames = [ "riverqueue", "@riverqueue/cli", "@riverqueue/driver-knex", + "@riverqueue/driver-mysql", "@riverqueue/driver-pg", "@riverqueue/driver-prisma", "@riverqueue/driver-sqlite", @@ -1066,7 +1076,7 @@ async function verifyExamples(archiveByPackageName) { } } -async function installTemporalSetup(directory) { +async function installTemporalSetup(directory, installArgs = []) { const packagePath = join(directory, "package.json"); const packageJson = JSON.parse(await readFile(packagePath, "utf8")); packageJson.dependencies = { @@ -1075,7 +1085,7 @@ async function installTemporalSetup(directory) { rootPackageJson.devDependencies["@js-temporal/polyfill"], }; await writeFile(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`); - await npmInstall(directory); + await npmInstall(directory, installArgs); await copyFile( resolve(repositoryRoot, "scripts", "temporal-setup.mjs"), join(directory, "temporal-setup.mjs") diff --git a/js/scripts/check-readme-snippets.mjs b/js/scripts/check-readme-snippets.mjs index 390f2b6c0..6b2a82de7 100644 --- a/js/scripts/check-readme-snippets.mjs +++ b/js/scripts/check-readme-snippets.mjs @@ -14,6 +14,7 @@ const readmes = [ .map((name) => `docs/${name}`), "cli/README.md", "driver/knex/README.md", + "driver/mysql/README.md", "driver/pg/README.md", "driver/prisma/README.md", "driver/sqlite/README.md", @@ -98,6 +99,9 @@ function checkProgram(directory, virtualFiles, readme) { "@riverqueue/driver-knex": [ resolve(repositoryRoot, "driver/knex/dist/index.d.ts"), ], + "@riverqueue/driver-mysql": [ + resolve(repositoryRoot, "driver/mysql/dist/index.d.ts"), + ], "@riverqueue/driver-pg": [ resolve(repositoryRoot, "driver/pg/dist/index.d.ts"), ], diff --git a/js/scripts/sync-migrations.mjs b/js/scripts/sync-migrations.mjs index 2373b05f1..9bed1cb2b 100644 --- a/js/scripts/sync-migrations.mjs +++ b/js/scripts/sync-migrations.mjs @@ -14,6 +14,7 @@ const riverRoot = resolve(repositoryRoot, ".."); const targetRoot = resolve(repositoryRoot, "migrate/migrations"); const check = process.argv.includes("--check"); const sources = { + mysql: "riverdriver/rivermysql/migration/main", postgres: "riverdriver/riverpgxv5/migration/main", sqlite: "riverdriver/riversqlite/migration/main", }; diff --git a/js/src/pilot.ts b/js/src/pilot.ts index 7aa882cf6..8d4a1431e 100644 --- a/js/src/pilot.ts +++ b/js/src/pilot.ts @@ -738,6 +738,11 @@ export type PilotFactory< * a `node:sqlite` database. */ export type DriverMigrationTarget = + | { + readonly backend: "mysql"; + readonly pool: object; + readonly schema: string | undefined; + } | { readonly client: object; readonly schema: string | undefined } | { readonly database: object; diff --git a/js/tsconfig.tests.json b/js/tsconfig.tests.json index f93892c2a..5fa933156 100644 --- a/js/tsconfig.tests.json +++ b/js/tsconfig.tests.json @@ -23,6 +23,9 @@ ], "@riverqueue/driver-knex": [ "./driver/knex/src/index.ts" + ], + "@riverqueue/driver-mysql": [ + "./driver/mysql/src/index.ts" ] } }, diff --git a/js/typedoc.json b/js/typedoc.json index f49a46f20..4b96390fe 100644 --- a/js/typedoc.json +++ b/js/typedoc.json @@ -4,6 +4,7 @@ "src/index.ts", "cli/src/index.ts", "driver/knex/src/index.ts", + "driver/mysql/src/index.ts", "driver/pg/src/index.ts", "driver/prisma/src/index.ts", "driver/sqlite/src/index.ts", diff --git a/riverdbtest/riverdbtest.go b/riverdbtest/riverdbtest.go index 64468ca9a..652e4d549 100644 --- a/riverdbtest/riverdbtest.go +++ b/riverdbtest/riverdbtest.go @@ -481,7 +481,8 @@ type TestTxOpts struct { // run using TestSchema. This is meant for environments where parallelism // doesn't work as well, like SQLite, which will emit "busy" errors when // multiple clients try to share a schema, even when they're in separate - // transactions. + // transactions. Also applies to MySQL, where InnoDB deadlocks are common + // when multiple transactions are sharing a database. DisableSchemaSharing bool // IsTestTxHelper should be set to true for if TestTx is being called from diff --git a/riverdriver/go.sum b/riverdriver/go.sum index 8c4ec9ede..75f65974a 100644 --- a/riverdriver/go.sum +++ b/riverdriver/go.sum @@ -1,5 +1,5 @@ -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= diff --git a/riverdriver/river_driver_interface.go b/riverdriver/river_driver_interface.go index f76adc29b..231f3e660 100644 --- a/riverdriver/river_driver_interface.go +++ b/riverdriver/river_driver_interface.go @@ -25,6 +25,7 @@ import ( const AllQueuesString = "*" const ( + DatabaseNameMySQL = "mysql" DatabaseNamePostgres = "postgres" DatabaseNameSQLite = "sqlite" ) @@ -53,7 +54,7 @@ var ( type Driver[TTx any] interface { // ArgPlaceholder is the placeholder character used in query positional // arguments, so "$" for "$1", "$2", "$3", etc. This is a "$" for Postgres - // and "?" for SQLite. + // and "?" for MySQL and SQLite. // // API is not stable. DO NOT USE. ArgPlaceholder() string @@ -128,6 +129,13 @@ type Driver[TTx any] interface { // API is not stable. DO NOT USE. PoolSet(dbPool any) error + // SafeIdentifier returns a safely quoted identifier (e.g. a table or + // schema name) for use in SQL queries. Each driver quotes using its + // native syntax: double quotes for Postgres/SQLite, backticks for MySQL. + // + // API is not stable. DO NOT USE. + SafeIdentifier(ident string) string + // SQLFragmentColumnContainsAll generates an SQL fragment to be included as // a predicate in a `WHERE` query for a collection column containing all of // the given values. Postgres uses array containment while SQLite compares @@ -144,6 +152,12 @@ type Driver[TTx any] interface { // API is not stable. DO NOT USE. SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) + // SQLFragmentColumnContainsJSON generates a structural JSON containment + // predicate and its named arguments. Column and namedArg are trusted SQL identifiers. + // + // API is not stable. DO NOT USE. + SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) + // SQLFragmentColumnIn generates an SQL fragment to be included as a // predicate in a `WHERE` query for the existence of a set of values in a // column like `id IN (...)`. The actual implementation depends on support diff --git a/riverdriver/riverdatabasesql/go.sum b/riverdriver/riverdatabasesql/go.sum index 83122b1b7..ec7802797 100644 --- a/riverdriver/riverdatabasesql/go.sum +++ b/riverdriver/riverdatabasesql/go.sum @@ -12,16 +12,16 @@ github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFr github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go b/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go index 0f7f8142f..b976c4046 100644 --- a/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go +++ b/riverdriver/riverdatabasesql/internal/dbsqlc/river_job.sql.go @@ -1524,7 +1524,9 @@ WITH job_input AS ( unnest($6::boolean[]) AS finalized_at_do_update, unnest($7::timestamptz[]) AS finalized_at, unnest($8::boolean[]) AS metadata_do_merge, - unnest($9::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest($9::text[])::jsonb AS metadata_updates, unnest($10::boolean[]) AS scheduled_at_do_update, unnest($11::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] diff --git a/riverdriver/riverdatabasesql/river_database_sql_driver.go b/riverdriver/riverdatabasesql/river_database_sql_driver.go index 0671e852f..be0b9b316 100644 --- a/riverdriver/riverdatabasesql/river_database_sql_driver.go +++ b/riverdriver/riverdatabasesql/river_database_sql_driver.go @@ -90,8 +90,9 @@ func NewWithPgxListener(dbPool *sql.DB, listenerPool *pgxpool.Pool) *Driver { const argPlaceholder = "$" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d} @@ -132,6 +133,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] return fmt.Sprintf("%s && @%s", column, namedArg), pq.Array(values), nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return column + " @> @" + namedArg + "::jsonb", map[string]any{namedArg: fragment}, nil +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { // Identical to the Pgx implementation except for use of `pg.Array`. return fmt.Sprintf("%s = any(@%s)", column, column), pq.Array(values), nil diff --git a/riverdriver/riverdrivertest/client_cancel_test.go b/riverdriver/riverdrivertest/client_cancel_test.go index 601c5c882..bb01449ba 100644 --- a/riverdriver/riverdrivertest/client_cancel_test.go +++ b/riverdriver/riverdrivertest/client_cancel_test.go @@ -24,6 +24,9 @@ func exerciseClientCancelRunningJob[TTx any](ctx context.Context, t *testing.T, config := newTestConfig(t, schema) config.FetchPollInterval = time.Minute + // Startup maintenance may briefly lock a job and make the first SKIP LOCKED + // fetch miss it. Keep this test focused on cancellation after that fetch. + config.LeaderElectionDisabled = true config.PollOnly = pollOnly config.Queues = map[string]river.QueueConfig{river.QueueDefault: {MaxWorkers: 1}} diff --git a/riverdriver/riverdrivertest/client_job_delete_test.go b/riverdriver/riverdrivertest/client_job_delete_test.go index 01fcc02a6..46d2ed261 100644 --- a/riverdriver/riverdrivertest/client_job_delete_test.go +++ b/riverdriver/riverdrivertest/client_job_delete_test.go @@ -58,7 +58,7 @@ func exerciseClientJobDeleteManyWhere[TTx any](ctx context.Context, t *testing.T ) predicate := "(args->>'customer_id')::bigint IN (@customer_a, @customer_b)" - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if bundle.driver.DatabaseName() != riverdriver.DatabaseNamePostgres { predicate = "args ->> '$.customer_id' IN (@customer_a, @customer_b)" } diff --git a/riverdriver/riverdrivertest/client_job_update_test.go b/riverdriver/riverdrivertest/client_job_update_test.go index e1b185e33..927565ce4 100644 --- a/riverdriver/riverdrivertest/client_job_update_test.go +++ b/riverdriver/riverdrivertest/client_job_update_test.go @@ -11,6 +11,7 @@ import ( "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/testfactory" "github.com/riverqueue/river/rivertype" ) @@ -155,4 +156,106 @@ func exerciseClientJobUpdate[TTx any](ctx context.Context, t *testing.T, require.Nil(t, loser.row.FinalizedAt, "the loser of the race condition must observe the winner's finalization clear, not its own snapshot's") }) + t.Run("JobScheduleSkipsLockedJobs", func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t) + if driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL skips locked scheduler candidates; other drivers may wait for them") + } + exec := driver.GetExecutor() + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + ScheduledAt: new(time.Now().Add(-time.Minute)), Schema: schema, State: new(rivertype.JobStateScheduled), + }) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ID: job.ID, Schema: schema}) + require.NoError(t, err) + + scheduleCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + rows, err := exec.JobSchedule(scheduleCtx, &riverdriver.JobScheduleParams{Max: 10, Schema: schema}) + require.NoError(t, err) + require.Empty(t, rows) + require.NoError(t, tx.Rollback(ctx)) + + rows, err = exec.JobSchedule(ctx, &riverdriver.JobScheduleParams{Max: 10, Schema: schema}) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, job.ID, rows[0].Job.ID) + }) + + // A MySQL transaction can already have a REPEATABLE READ snapshot when it + // invokes a state change. No-op updates must still return the current row. + t.Run("JobUpdateWithExistingSnapshot", func(t *testing.T) { + t.Parallel() + + for _, operation := range []string{"Cancel", "DeleteMany", "Retry", "SetStateIfRunning"} { + t.Run(operation, func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t) + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite does not permit a second writer with an open read transaction") + } + exec := driver.GetExecutor() + state := rivertype.JobStateScheduled + switch operation { + case "Retry": + state = rivertype.JobStateCancelled + case "SetStateIfRunning": + state = rivertype.JobStateRunning + } + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Schema: schema, State: &state}) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: job.ID, Schema: schema}) + require.NoError(t, err) + + var current *rivertype.JobRow + switch operation { + case "Cancel": + params := &riverdriver.JobCancelParams{ID: job.ID, CancelAttemptedAt: time.Now(), Schema: schema} + _, err = exec.JobCancel(ctx, params) + require.NoError(t, err) + current, err = tx.JobCancel(ctx, params) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateCancelled, current.State) + case "DeleteMany": + _, err = exec.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ + ID: job.ID, Schema: schema, State: rivertype.JobStateRunning, StateDoUpdate: true, + }) + require.NoError(t, err) + rows, err := tx.JobDeleteMany(ctx, &riverdriver.JobDeleteManyParams{ + Max: 10, OrderByClause: "id", Schema: schema, WhereClause: "true", + }) + require.NoError(t, err) + require.Empty(t, rows, "jobs that are now running must not be deleted") + case "Retry": + params := &riverdriver.JobRetryParams{ID: job.ID, Schema: schema} + _, err = exec.JobRetry(ctx, params) + require.NoError(t, err) + current, err = tx.JobRetry(ctx, params) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateAvailable, current.State) + case "SetStateIfRunning": + _, err = exec.JobUpdateFull(ctx, &riverdriver.JobUpdateFullParams{ + ID: job.ID, FinalizedAt: new(time.Now()), FinalizedAtDoUpdate: true, Schema: schema, + State: rivertype.JobStateCompleted, StateDoUpdate: true, + }) + require.NoError(t, err) + rows, err := tx.JobSetStateIfRunningMany(ctx, &riverdriver.JobSetStateIfRunningManyParams{ + ID: []int64{job.ID}, Attempt: []*int{nil}, ErrData: [][]byte{nil}, FinalizedAt: []*time.Time{new(time.Now())}, + MetadataDoMerge: []bool{false}, MetadataUpdates: [][]byte{nil}, ScheduledAt: []*time.Time{nil}, + Schema: schema, State: []rivertype.JobState{rivertype.JobStateCancelled}, + }) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, rivertype.JobStateCompleted, rows[0].State) + } + }) + } + }) } diff --git a/riverdriver/riverdrivertest/driver_client_test.go b/riverdriver/riverdrivertest/driver_client_test.go index 1afb65832..9e6662147 100644 --- a/riverdriver/riverdrivertest/driver_client_test.go +++ b/riverdriver/riverdrivertest/driver_client_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + _ "github.com/go-sql-driver/mysql" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/stdlib" "github.com/lib/pq" @@ -25,6 +26,7 @@ import ( "github.com/riverqueue/river/riverdbtest" "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/riverdriver/riverdatabasesql" + "github.com/riverqueue/river/riverdriver/rivermysql" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riversqlite" "github.com/riverqueue/river/rivershared/riversharedtest" @@ -191,6 +193,26 @@ func TestClientWithDriverRiverLibSQL(t *testing.T) { ) } +func TestClientWithDriverRiverMySQL(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = context.Background() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = rivermysql.New(dbPool) + ) + + ExerciseClient(ctx, t, + func(ctx context.Context, t *testing.T) (riverdriver.Driver[*sql.Tx], string) { + t.Helper() + + return driver, riverdbtest.TestSchema(ctx, t, driver, nil) + }, + ) +} + func TestClientWithDriverRiverSQLiteModernC(t *testing.T) { t.Parallel() @@ -443,7 +465,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, Schema: bundle.schema, }) require.NoError(t, getErr) - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if bundle.driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -462,7 +484,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, {Args: noOpArgs{Name: "same"}, InsertOpts: &river.InsertOpts{UniqueOpts: river.UniqueOpts{ByArgs: true}}}, {Args: noOpArgs{Name: "same"}, InsertOpts: &river.InsertOpts{UniqueOpts: river.UniqueOpts{ByArgs: true}}}, }) - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if bundle.driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -701,6 +723,39 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("JobDeleteBeforeSkipsLocked", func(t *testing.T) { + t.Parallel() + + _, bundle := setup(t) + if bundle.driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL's cleaner skips locked expired jobs to avoid secondary-index deadlocks") + } + now := time.Now().UTC() + running := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{Schema: bundle.schema, State: new(rivertype.JobStateRunning)}) + locked := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + unlocked := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + // A full batch must also avoid a DELETE table scan through locked rows. + for range 999 { + testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), Schema: bundle.schema, State: new(rivertype.JobStateCompleted)}) + } + _, execTx := beginTx(ctx, t, bundle) + for _, id := range []int64{running.ID, locked.ID} { + _, err := execTx.JobUpdate(ctx, &riverdriver.JobUpdateParams{ID: id, Metadata: []byte(`{"locked":true}`), MetadataDoMerge: true, Schema: bundle.schema}) + require.NoError(t, err) + } + cleanupCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + deleted, err := bundle.exec.JobDeleteBefore(cleanupCtx, &riverdriver.JobDeleteBeforeParams{CompletedDoDelete: true, CompletedFinalizedAtHorizon: now, Max: 1000, Schema: bundle.schema}) + require.NoError(t, err) + require.Equal(t, 1000, deleted) + _, err = bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: unlocked.ID, Schema: bundle.schema}) + require.ErrorIs(t, err, rivertype.ErrNotFound) + for _, id := range []int64{running.ID, locked.ID} { + _, err := bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: id, Schema: bundle.schema}) + require.NoError(t, err) + } + }) + t.Run("JobDeleteManyUnsafeAll", func(t *testing.T) { t.Parallel() @@ -1156,6 +1211,56 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, require.Equal(t, job.ID, listRes.Jobs[0].ID) }) + t.Run("JobListMetadataStructure", func(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + metadata string + fragment string + matches bool + }{ + {"ArrayNestingMatches", `{"array":[[1,2]]}`, `{"array":[[1]]}`, true}, + {"ArrayNestingMismatch", `{"array":[[1]]}`, `{"array":[1]}`, false}, + {"ArrayObjectMatches", `{"array":[{"a":1,"b":2}]}`, `{"array":[{"b":2,"a":1}]}`, true}, + {"ArrayObjectSplit", `{"array":[{"a":1},{"b":2}]}`, `{"array":[{"a":1,"b":2}]}`, false}, + {"ArrayScalarMismatch", `{"array":[1]}`, `{"array":1}`, false}, + {"EmptyArrayMatches", `{"array":[1]}`, `{"array":[]}`, true}, + {"EmptyContainerMismatch", `{"array":{}}`, `{"array":[]}`, false}, + {"LargeIntegerMatches", `{"number":9007199254740993}`, `{"number":9007199254740993}`, true}, + {"LargeIntegerMismatch", `{"number":9007199254740993}`, `{"number":9007199254740992}`, false}, + {"NullMatches", `{"value":null}`, `{"value":null}`, true}, + {"NullMissing", `{}`, `{"value":null}`, false}, + {"QuotedKey", `{"a.b\\c\"d'e":true}`, `{"a.b\\c\"d'e":true}`, true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + client, bundle := setup(t) + if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("Metadata containment is not supported on SQLite") + } + job := testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{ + Metadata: []byte(testCase.metadata), + Schema: bundle.schema, + }) + params := river.NewJobListParams().Metadata(testCase.fragment) + result, err := client.JobList(ctx, params) + require.NoError(t, err) + wantIDs := []int64{} + if testCase.matches { + wantIDs = []int64{job.ID} + } + require.Equal(t, wantIDs, sliceutil.Map(result.Jobs, func(job *rivertype.JobRow) int64 { return job.ID })) + + tx, _ := beginTx(ctx, t, bundle) + result, err = client.JobListTx(ctx, tx, params) + require.NoError(t, err) + require.Equal(t, wantIDs, sliceutil.Map(result.Jobs, func(job *rivertype.JobRow) int64 { return job.ID })) + }) + } + }) + t.Run("JobListMixedStatePagination", func(t *testing.T) { t.Parallel() @@ -1566,9 +1671,12 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, listParams := river.NewJobListParams() - if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + switch bundle.driver.DatabaseName() { + case riverdriver.DatabaseNameSQLite: listParams = listParams.Where("metadata ->> @json_path = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": "bar"}) - } else { + case riverdriver.DatabaseNameMySQL: + listParams = listParams.Where("JSON_UNQUOTE(JSON_EXTRACT(metadata, @json_path)) = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": "bar"}) + default: // "bar" is quoted in this branch because `jsonb_path_query_first` needs to be compared to a JSON value listParams = listParams.Where("jsonb_path_query_first(metadata, @json_path) = @json_val", river.NamedArgs{"json_path": "$.foo", "json_val": `"bar"`}) } @@ -2001,7 +2109,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, // The invalid value is left in place, and once repaired, the job // shows the failed attempt. require.Equal(t, sqliteMalformedValue, sqliteJobColumnText(ctx, t, bundle.exec, jobID, column)) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, jobID, column, repairedValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, jobID, column, repairedValue) } job, err := client.JobGet(ctx, jobID) @@ -2028,8 +2136,8 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, t.Parallel() config, bundle := setupConfig(t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } config.RetryPolicy = &retryPolicyAnHourLater{} @@ -2043,8 +2151,8 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, goodJob2 = testfactory.Job(ctx, t, bundle.exec, &testfactory.JobOpts{Kind: new(noOpArgs{}.Kind()), Schema: bundle.schema}) undecodableValue = `{"not":"an array"}` ) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob1.ID, "tags", undecodableValue) - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob2.ID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob1.ID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob2.ID, "tags", undecodableValue) subscribeChan := subscribe(t, client) startClient(ctx, t, client) @@ -2070,7 +2178,7 @@ func ExerciseClient[TTx any](ctx context.Context, t *testing.T, require.ErrorContains(t, err, "error unmarshaling `tags`") // Once repaired, the job shows the failed attempt. - sqliteSetJobJSONColumn(ctx, t, bundle.exec, undecodableJob1.ID, "tags", `["tag"]`) + setJobJSONColumn(ctx, t, bundle.exec, bundle.driver, bundle.schema, undecodableJob1.ID, "tags", `["tag"]`) job, err := client.JobGet(ctx, undecodableJob1.ID) require.NoError(t, err) require.Equal(t, 1, job.Attempt) diff --git a/riverdriver/riverdrivertest/driver_test.go b/riverdriver/riverdrivertest/driver_test.go index 80abb865d..9d2922725 100644 --- a/riverdriver/riverdrivertest/driver_test.go +++ b/riverdriver/riverdrivertest/driver_test.go @@ -1,13 +1,17 @@ package riverdrivertest_test import ( + "cmp" "context" "database/sql" + "os" "runtime" "strconv" + "strings" "testing" "time" + "github.com/go-sql-driver/mysql" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" @@ -22,6 +26,7 @@ import ( "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/riverdriver/riverdatabasesql" "github.com/riverqueue/river/riverdriver/riverdrivertest" + "github.com/riverqueue/river/riverdriver/rivermysql" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riversqlite" "github.com/riverqueue/river/rivershared/riversharedtest" @@ -234,6 +239,62 @@ func TestDriverRiverLiteLibSQL(t *testing.T) { //nolint:dupl }) } +func TestDriverRiverMySQL(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + ctx := context.Background() + exerciseDriverMySQL(ctx, t, riversharedtest.DBPoolMySQL(ctx, t)) +} + +func TestDriverRiverMySQLInterpolateParams(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + config, err := mysql.ParseDSN(cmp.Or(os.Getenv("TEST_MYSQL_URL"), + "root@tcp(localhost:3306)/?parseTime=true&multiStatements=true&loc=UTC&time_zone=%27%2B00%3A00%27")) + require.NoError(t, err) + config.InterpolateParams = true + connector, err := mysql.NewConnector(config) + require.NoError(t, err) + pool := sql.OpenDB(connector) + t.Cleanup(func() { require.NoError(t, pool.Close()) }) + exerciseDriverMySQL(context.Background(), t, pool) +} + +func exerciseDriverMySQL(ctx context.Context, t *testing.T, pool *sql.DB) { + t.Helper() + + driver := rivermysql.New(pool) + + riverdrivertest.Exercise(ctx, t, + func(ctx context.Context, t *testing.T, opts *riverdbtest.TestSchemaOpts) (riverdriver.Driver[*sql.Tx], string) { + t.Helper() + + return driver, riverdbtest.TestSchema(ctx, t, driver, opts) + }, + func(ctx context.Context, t *testing.T) (riverdriver.Executor, riverdriver.Driver[*sql.Tx]) { + t.Helper() + + tx, schema := riverdbtest.TestTx(ctx, t, driver, &riverdbtest.TestTxOpts{ + // Disable schema sharing to reduce InnoDB deadlocks from + // parallel tests contending on the same database. + DisableSchemaSharing: true, + }) + + // MySQL has no search_path equivalent, so USE the test schema + // database so that unqualified queries resolve correctly. + if schema != "" { + _, err := tx.ExecContext(ctx, "USE `"+strings.ReplaceAll(schema, "`", "``")+"`") //nolint:gosec // The identifier is quoted and supplied by TestTx. + require.NoError(t, err) + } + + return driver.UnwrapExecutor(tx), driver + }) +} + func TestDriverRiverSQLiteModernC(t *testing.T) { //nolint:dupl t.Parallel() diff --git a/riverdriver/riverdrivertest/executor_tx.go b/riverdriver/riverdrivertest/executor_tx.go index 65e1372db..4115a1f35 100644 --- a/riverdriver/riverdrivertest/executor_tx.go +++ b/riverdriver/riverdrivertest/executor_tx.go @@ -212,7 +212,13 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, exec := setup(ctx, t) - require.NoError(t, exec.Exec(ctx, "SELECT $1 || $2", "foo", "bar")) + _, driver := executorWithTx(ctx, t) + switch driver.DatabaseName() { + case riverdriver.DatabaseNameMySQL: + require.NoError(t, exec.Exec(ctx, "SELECT CONCAT(?, ?)", "foo", "bar")) + default: + require.NoError(t, exec.Exec(ctx, "SELECT $1 || $2", "foo", "bar")) + } }) }) @@ -221,9 +227,8 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, { driver, _ := driverWithSchema(ctx, t, nil) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - t.Logf("Skipping PGAdvisoryXactLock test for SQLite") - return + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite || driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + t.Skipf("Skipping PGAdvisoryXactLock test for %s", driver.DatabaseName()) } } @@ -290,6 +295,44 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, require.Equal(t, "foo", fieldFoo) }) + t.Run("QueueMutationsWithExistingSnapshot", func(t *testing.T) { + t.Parallel() + + for _, operation := range []string{"Pause", "Resume", "Update"} { + t.Run(operation, func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite does not permit a second writer with an open read transaction") + } + exec := driver.GetExecutor() + queue := testfactory.Queue(ctx, t, exec, &testfactory.QueueOpts{Schema: schema}) + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.QueueGet(ctx, &riverdriver.QueueGetParams{Name: queue.Name, Schema: schema}) + require.NoError(t, err) + + deleted, err := exec.QueueDeleteExpired(ctx, &riverdriver.QueueDeleteExpiredParams{ + Max: 10, Schema: schema, UpdatedAtHorizon: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + require.Equal(t, []string{queue.Name}, deleted) + + switch operation { + case "Pause": + err = tx.QueuePause(ctx, &riverdriver.QueuePauseParams{Name: queue.Name, Schema: schema}) + case "Resume": + err = tx.QueueResume(ctx, &riverdriver.QueueResumeParams{Name: queue.Name, Schema: schema}) + case "Update": + _, err = tx.QueueUpdate(ctx, &riverdriver.QueueUpdateParams{Metadata: []byte(`{}`), Name: queue.Name, Schema: schema}) + } + require.ErrorIs(t, err, rivertype.ErrNotFound) + }) + } + }) + t.Run("WithTx", func(t *testing.T) { t.Parallel() @@ -371,6 +414,32 @@ func exerciseExecutorTx[TTx any](ctx context.Context, t *testing.T, }) } + t.Run("Savepoints", func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("uses MySQL session statistics to detect hidden savepoints") + } + tx, err := driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + + var name string + var before, after int + require.NoError(t, tx.QueryRow(ctx, "SHOW SESSION STATUS LIKE 'Com_savepoint'").Scan(&name, &before)) + require.NoError(t, dbutil.WithTx(ctx, tx, func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + _ = testfactory.Job(ctx, t, execTx, &testfactory.JobOpts{Schema: schema}) + return nil + })) + require.NoError(t, tx.QueryRow(ctx, "SHOW SESSION STATUS LIKE 'Com_savepoint'").Scan(&name, &after)) + if legacySubtransactions { + require.Greater(t, after, before, "legacy mode creates savepoints") + } else { + require.Equal(t, before, after, "driver internals must reuse the caller's transaction without savepoints") + } + }) + t.Run("TransactionIDs", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/riverdrivertest/go.mod b/riverdriver/riverdrivertest/go.mod index edb130bf9..4bfd0f302 100644 --- a/riverdriver/riverdrivertest/go.mod +++ b/riverdriver/riverdrivertest/go.mod @@ -6,12 +6,14 @@ toolchain go1.26.6 require ( github.com/davecgh/go-spew v1.1.1 + github.com/go-sql-driver/mysql v1.10.0 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pgx/v5 v5.11.0 github.com/lib/pq v1.12.3 github.com/riverqueue/river v0.51.1 github.com/riverqueue/river/riverdriver v0.51.1 github.com/riverqueue/river/riverdriver/riverdatabasesql v0.51.1 + github.com/riverqueue/river/riverdriver/rivermysql v0.0.0-00010101000000-000000000000 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 github.com/riverqueue/river/riverdriver/riversqlite v0.51.1 github.com/riverqueue/river/rivershared v0.51.1 @@ -25,7 +27,12 @@ require ( turso.tech/database/tursogo v0.8.3 ) +// Remove this replacement after rivermysql has had its first coordinated +// release. It lets go mod tidy resolve the new sibling module in the meantime. +replace github.com/riverqueue/river/riverdriver/rivermysql => ../rivermysql + require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/coder/websocket v1.8.12 // indirect github.com/dustin/go-humanize v1.1.0 // indirect diff --git a/riverdriver/riverdrivertest/go.sum b/riverdriver/riverdrivertest/go.sum index ec30cd38d..3719731ea 100644 --- a/riverdriver/riverdrivertest/go.sum +++ b/riverdriver/riverdrivertest/go.sum @@ -1,3 +1,5 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= @@ -9,6 +11,8 @@ github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2 github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -36,20 +40,20 @@ github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2K github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverdatabasesql v0.51.0 h1:k6IJTgJKuUr9EGi1DsEPg8nM/VvmRkDt60UP3CjCGH4= -github.com/riverqueue/river/riverdriver/riverdatabasesql v0.51.0/go.mod h1:ciOtbdc5VgqwpU4tAh8zk7IukHHlxbmesCI9RS4st78= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/riverdriver/riversqlite v0.51.0 h1:mWm7sg8L2U/cF2CdgQSOC1kX90S9sWomMZZgwHPD1Gs= -github.com/riverqueue/river/riverdriver/riversqlite v0.51.0/go.mod h1:akyTyxi/+4W0ND9QJzKOtxq880T0LFoRI5KlKILmdWs= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverdatabasesql v0.51.1 h1:T16D1SBH3+wzjPZ1DQcX0nje5nsblzoK6dUhQp2zIhw= +github.com/riverqueue/river/riverdriver/riverdatabasesql v0.51.1/go.mod h1:LwwMwYmNtvg3xnIseo5ucTvOMKedzuxxDRKlFELz/Xo= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/riverdriver/riversqlite v0.51.1 h1:VmAPT1AETH3sqbbHG9GPiH0OmQFXJk+hFeRgCVUKoI0= +github.com/riverqueue/river/riverdriver/riversqlite v0.51.1/go.mod h1:G6NSbOzZO4eJwvtIHzwh9i4tAzKwoUrJ25AzkZpFwFY= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/riverdriver/riverdrivertest/job_delete.go b/riverdriver/riverdrivertest/job_delete.go index dd8a14dd2..86084584f 100644 --- a/riverdriver/riverdrivertest/job_delete.go +++ b/riverdriver/riverdrivertest/job_delete.go @@ -137,6 +137,31 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT afterHorizon = horizon.Add(1 * time.Minute) ) + t.Run("RespectsDisabledStates", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + cancelledJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) + completedJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) + discardedJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) + + numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ + CompletedDoDelete: true, + CompletedFinalizedAtHorizon: horizon, + Max: 1_000, + }) + require.NoError(t, err) + require.Equal(t, 1, numDeleted) + + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: cancelledJob.ID}) + require.NoError(t, err) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: completedJob.ID}) + require.ErrorIs(t, err, rivertype.ErrNotFound) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: discardedJob.ID}) + require.NoError(t, err) + }) + t.Run("Success", func(t *testing.T) { t.Parallel() @@ -235,16 +260,17 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT exec, _ := setup(ctx, t) var ( - cancelledJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) - completedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) - discardedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) - excludedQueue1 = "excluded1" excludedQueue2 = "excluded2" - // Not deleted because in an omitted queue. + // Insert excluded jobs first to verify that they don't consume the + // limited candidate batch and starve later eligible jobs. notDeletedJob1 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &excludedQueue1, State: new(rivertype.JobStateCompleted)}) notDeletedJob2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &excludedQueue2, State: new(rivertype.JobStateCompleted)}) + + cancelledJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCancelled)}) + completedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateCompleted)}) + discardedJob = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, State: new(rivertype.JobStateDiscarded)}) ) numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ @@ -254,7 +280,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT CompletedFinalizedAtHorizon: horizon, DiscardedDoDelete: true, DiscardedFinalizedAtHorizon: horizon, - Max: 1_000, + Max: 3, QueuesExcluded: []string{excludedQueue1, excludedQueue2}, }) require.NoError(t, err) @@ -373,6 +399,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT deletedJob2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: &beforeHorizon, Queue: &includedQueue2, State: new(rivertype.JobStateCompleted)}) ) + // The three older non-included jobs must not consume this batch. numDeleted, err := exec.JobDeleteBefore(ctx, &riverdriver.JobDeleteBeforeParams{ CancelledDoDelete: true, CancelledFinalizedAtHorizon: horizon, @@ -380,7 +407,7 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT CompletedFinalizedAtHorizon: horizon, DiscardedDoDelete: true, DiscardedFinalizedAtHorizon: horizon, - Max: 1_000, + Max: 2, QueuesIncluded: []string{includedQueue1, includedQueue2}, }) require.NoError(t, err) @@ -550,5 +577,30 @@ func exerciseJobDelete[TTx any](ctx context.Context, t *testing.T, executorWithT require.NoError(t, err) require.Equal(t, []int64{job1.ID, job2.ID, job3.ID, job4.ID, job5.ID}, sliceutil.Map(deletedJobs, func(j *rivertype.JobRow) int64 { return j.ID })) }) + + t.Run("SortedResultsDescending", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + if bundle.driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skip("SQLite always returns JobDeleteMany results ordered by ID") + } + + var ( + job1 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job2 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job3 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job4 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + job5 = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) + ) + + deletedJobs, err := exec.JobDeleteMany(ctx, &riverdriver.JobDeleteManyParams{ + Max: 100, + OrderByClause: "id DESC", + WhereClause: "true", + }) + require.NoError(t, err) + require.Equal(t, []int64{job5.ID, job4.ID, job3.ID, job2.ID, job1.ID}, sliceutil.Map(deletedJobs, func(j *rivertype.JobRow) int64 { return j.ID })) + }) }) } diff --git a/riverdriver/riverdrivertest/job_insert.go b/riverdriver/riverdrivertest/job_insert.go index 172f2b971..44962b558 100644 --- a/riverdriver/riverdrivertest/job_insert.go +++ b/riverdriver/riverdrivertest/job_insert.go @@ -93,9 +93,8 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, uniqueNonces[nonce] = true } - // SQLite needs to set a special metadata key to be able to - // check for duplicates. Remove this for purposes of comparing - // inserted metadata. + // Some drivers use a special metadata key to detect duplicates. + // Remove it for purposes of comparing inserted metadata. job.Metadata, err = sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) require.NoError(t, err) @@ -120,6 +119,73 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("BatchWithoutUniqueKeysPreservesInput", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + + now := time.Now().UTC().Truncate(bundle.driver.TimePrecision()) + params := &riverdriver.JobInsertFastManyParams{} + for i := range 50 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + CreatedAt: new(now.Add(time.Duration(i) * time.Second)), + EncodedArgs: fmt.Appendf(nil, `{"i":%d}`, i), + Kind: "batch_plain", + MaxAttempts: 25, + Metadata: fmt.Appendf(nil, `{"i":%d,"text":"Ă© <&> \"quoted\""}`, i), + Priority: 1, + Queue: "default", + ScheduledAt: new(now.Add(time.Duration(i) * time.Minute)), + State: rivertype.JobStateAvailable, + Tags: []string{"one", "two"}, + }) + } + rows, err := exec.JobInsertFastMany(ctx, params) + require.NoError(t, err) + require.Len(t, rows, len(params.Jobs)) + for i, result := range rows { + input := params.Jobs[i] + require.False(t, result.UniqueSkippedAsDuplicate) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: result.Job.ID}) + require.NoError(t, err) + require.Equal(t, result.Job, stored) + metadata, err := sjson.DeleteBytes(stored.Metadata, riverdriver.UniqueInsertMetadataKey) + require.NoError(t, err) + require.JSONEq(t, string(input.Metadata), string(metadata)) + require.JSONEq(t, string(input.EncodedArgs), string(stored.EncodedArgs)) + require.Equal(t, *input.CreatedAt, stored.CreatedAt) + require.Equal(t, *input.ScheduledAt, stored.ScheduledAt) + require.Equal(t, input.Tags, stored.Tags) + } + }) + + t.Run("DuplicateIDWithoutUniqueKeyErrors", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + id := rand.Int64() + params := &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{ + { + EncodedArgs: []byte(`{}`), + ID: &id, + Kind: "test_kind", + MaxAttempts: rivercommon.MaxAttemptsDefault, + Priority: rivercommon.PriorityDefault, + Queue: rivercommon.QueueDefault, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + }, + } + + _, err := exec.JobInsertFastMany(ctx, params) + require.NoError(t, err) + + _, err = exec.JobInsertFastMany(ctx, params) + require.Error(t, err) + }) + t.Run("MissingValuesDefaultAsExpected", func(t *testing.T) { t.Parallel() @@ -186,6 +252,62 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("MixedNewAndDuplicateJobs", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + params := &riverdriver.JobInsertFastManyParams{} + for i := range 6 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Kind: "test_kind", MaxAttempts: 25, Metadata: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}, + }) + } + for _, i := range []byte{0, 2, 4} { + // Include bytes that aren't UTF-8 and differ only after a NUL. + params.Jobs[i].UniqueKey = []byte{0xff, 0x00, i, 0x80} + params.Jobs[i].UniqueStates = 0xff + } + params.Jobs[3].UniqueKey = params.Jobs[0].UniqueKey + params.Jobs[3].UniqueStates = 1 << 2 // Inactive for an available job. + + // Seed in reverse order, so returning rows sorted by ID would be wrong. + seed, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{params.Jobs[4], params.Jobs[0]}, + }) + require.NoError(t, err) + results, err := exec.JobInsertFastMany(ctx, params) + require.NoError(t, err) + require.Len(t, results, len(params.Jobs)) + require.Equal(t, seed[1].Job.ID, results[0].Job.ID) + require.Equal(t, seed[0].Job.ID, results[4].Job.ID) + for i, result := range results { + require.Equal(t, i == 0 || i == 4, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + require.Equal(t, params.Jobs[i].UniqueKey, result.Job.UniqueKey) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: result.Job.ID}) + require.NoError(t, err) + for _, job := range []*rivertype.JobRow{result.Job, stored} { + metadata, err := sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) + require.NoError(t, err) + require.JSONEq(t, string(params.Jobs[i].Metadata), string(metadata)) + } + } + + // No new IDs are generated when every job conflicts. + duplicates, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{params.Jobs[4], params.Jobs[2], params.Jobs[0]}, + }) + require.NoError(t, err) + require.Len(t, duplicates, 3) + for i, original := range []int{4, 2, 0} { + require.True(t, duplicates[i].UniqueSkippedAsDuplicate) + require.Equal(t, results[original].Job.ID, duplicates[i].Job.ID) + } + }) + t.Run("UniqueConflict", func(t *testing.T) { t.Parallel() @@ -300,7 +422,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, Jobs: []*riverdriver.JobInsertFastParams{job, job}, Schema: schema, }) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if driver.DatabaseName() != riverdriver.DatabaseNamePostgres { require.ErrorContains(t, err, "unique key appears more than once in batch") } else { require.ErrorContains(t, err, "ON CONFLICT DO UPDATE command cannot affect row a second time") @@ -449,6 +571,39 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, } }) + t.Run("BatchWithoutUniqueKeys", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + now := time.Date(2026, 10, 15, 12, 30, 1, 123456000, time.FixedZone("offset", -5*60*60)) + params := &riverdriver.JobInsertFastManyParams{} + for i := range 3 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + CreatedAt: &now, EncodedArgs: []byte(`{"id":123456789,"text":"é🌊\n\"quoted\""}`), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`{"id":123456789}`), + Priority: i + 1, Queue: "default", ScheduledAt: &now, + State: rivertype.JobStateAvailable, Tags: []string{"é🌊", `"quoted"`}, + }) + } + + count, err := exec.JobInsertFastManyNoReturning(ctx, params) + require.NoError(t, err) + require.Equal(t, len(params.Jobs), count) + jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{Kind: []string{"batch"}}) + require.NoError(t, err) + require.Len(t, jobs, len(params.Jobs)) + for i, job := range jobs { + require.Contains(t, string(job.EncodedArgs), "123456789") + require.Contains(t, string(job.Metadata), "123456789") + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(job.EncodedArgs)) + require.Equal(t, params.Jobs[i].Tags, job.Tags) + require.Equal(t, i+1, job.Priority) + require.WithinDuration(t, now, job.CreatedAt, bundle.driver.TimePrecision()) + require.WithinDuration(t, now, job.ScheduledAt, bundle.driver.TimePrecision()) + require.Nil(t, job.UniqueKey) + } + }) + t.Run("MissingValuesDefaultAsExpected", func(t *testing.T) { t.Parallel() @@ -837,7 +992,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, _, err := exec.JobInsertFull(ctx, params) require.Error(t, err) // two separate error messages here for Postgres and SQLite - require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null")`, err.Error()) + require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null"|Check constraint 'finalized_or_finalized_at_null' is violated)`, err.Error()) }) t.Run(fmt.Sprintf("CanSetState%sWithFinalizedAt", capitalizeJobState(state)), func(t *testing.T) { @@ -887,7 +1042,7 @@ func exerciseJobInsert[TTx any](ctx context.Context, t *testing.T, })) require.Error(t, err) // two separate error messages here for Postgres and SQLite - require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null")`, err.Error()) + require.Regexp(t, `(CHECK constraint failed: finalized_or_finalized_at_null|violates check constraint "finalized_or_finalized_at_null"|Check constraint 'finalized_or_finalized_at_null' is violated)`, err.Error()) }) } }) diff --git a/riverdriver/riverdrivertest/job_read.go b/riverdriver/riverdrivertest/job_read.go index 05037a26a..7ea5479ee 100644 --- a/riverdriver/riverdrivertest/job_read.go +++ b/riverdriver/riverdrivertest/job_read.go @@ -2,6 +2,7 @@ package riverdrivertest import ( "context" + "fmt" "sort" "strconv" "testing" @@ -278,6 +279,8 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx exec, _ := setup(ctx, t) unknown := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("unknown"), Priority: new(1)}) + unknownCase := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("KNOWN1"), Priority: new(1)}) + unknownSpace := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1 "), Priority: new(1)}) known1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1"), Priority: new(2)}) known2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known2"), Priority: new(3)}) otherQueue := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("known1"), Priority: new(1), Queue: new("other")}) @@ -298,7 +301,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx require.Equal(t, rivertype.JobStateRunning, jobs[0].State) } - for _, expected := range []*rivertype.JobRow{unknown, otherQueue} { + for _, expected := range []*rivertype.JobRow{unknown, unknownCase, unknownSpace, otherQueue} { job, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: expected.ID}) require.NoError(t, err) require.Equal(t, expected, job) @@ -616,14 +619,58 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx } }) + t.Run("ReturnedRowsMatchStored", func(t *testing.T) { + t.Parallel() + + for _, customNow := range []bool{false, true} { + t.Run(fmt.Sprintf("CustomNow=%t", customNow), func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + + var now *time.Time + if customNow { + now = new(time.Now().UTC().Add(time.Minute).Truncate(bundle.driver.TimePrecision())) + } + for i := range 3 { + testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Attempt: new(i + 1), + AttemptedBy: []string{"first", "second", "third"}, + EncodedArgs: []byte(`{"large":9007199254740993}`), + Metadata: []byte(`{"keep":true}`), + Tags: []string{"tag"}, + }) + } + jobs := getAvailable(t, exec, &riverdriver.JobGetAvailableParams{ + ClientID: testClientID, + MaxAttemptedBy: maxAttemptedBy, + MaxToLock: maxToLock, + Now: now, + Queue: rivercommon.QueueDefault, + }) + require.Len(t, jobs, 3) + for _, job := range jobs { + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: job.ID}) + require.NoError(t, err) + require.Equal(t, stored, job) + require.Equal(t, []string{"first", "second", "third", testClientID}, job.AttemptedBy) + require.Equal(t, rivertype.JobStateRunning, job.State) + if now != nil { + require.Equal(t, now, job.AttemptedAt) + } + } + }) + } + }) + // Every locked job is returned, with decode errors keyed by job ID so // the caller can fail only the attempts whose rows couldn't be decoded. t.Run("UndecodableJobsReturnedWithDecodeErrors", func(t *testing.T) { t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } job1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) @@ -631,9 +678,9 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx job3 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) job4 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{}) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "errors", `{"not":"an array"}`) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "tags", `{"not":"an array"}`) - sqliteSetJobJSONColumn(ctx, t, exec, job4.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "errors", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job4.ID, "tags", `{"not":"an array"}`) res, err := exec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ ClientID: testClientID, @@ -725,6 +772,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx job2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("kind2")}) // Not returned. + _ = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("KIND1")}) _ = testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Kind: new("kind3")}) jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{ @@ -902,8 +950,8 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } var ( @@ -914,7 +962,7 @@ func exerciseJobRead[TTx any](ctx context.Context, t *testing.T, executorWithTx stuckJob1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &beforeHorizon, State: new(rivertype.JobStateRunning)}) stuckJob2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &beforeHorizon, State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, stuckJob1.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", stuckJob1.ID, "tags", `{"not":"an array"}`) stuckJobs, err := exec.JobGetStuck(ctx, &riverdriver.JobGetStuckParams{ Max: 10, diff --git a/riverdriver/riverdrivertest/job_update.go b/riverdriver/riverdrivertest/job_update.go index e537229a5..840dc9b01 100644 --- a/riverdriver/riverdrivertest/job_update.go +++ b/riverdriver/riverdrivertest/job_update.go @@ -367,15 +367,15 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold a non-array errors value") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime attemptedAt := now.Add(-time.Hour) badJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &attemptedAt, State: new(rivertype.JobStateRunning)}) goodJob := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{AttemptedAt: &attemptedAt, State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, badJob.ID, "errors", `{"error":"existing value"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", badJob.ID, "errors", `{"error":"existing value"}`) _, err := exec.JobRescueMany(ctx, &riverdriver.JobRescueManyParams{ ID: []int64{badJob.ID, goodJob.ID}, @@ -883,8 +883,8 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } var ( @@ -903,7 +903,7 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT undecodableJobIDs = []int64{undecodableJob.ID, conflictingJob.ID, scheduledJob.ID} ) for _, jobID := range undecodableJobIDs { - sqliteSetJobJSONColumn(ctx, t, exec, jobID, "tags", undecodableValue) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", jobID, "tags", undecodableValue) } // Conflicts with conflictingJob, which is discarded instead of scheduled. @@ -1024,6 +1024,42 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT require.Equal(t, "unique-key", string(jobUpdated.UniqueKey)) }) + t.Run("CompletesBatchWithoutChangingOtherFields", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + + now := precisionTestTime.Truncate(bundle.driver.TimePrecision()) + original := []*rivertype.JobRow{ + testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Metadata: []byte(`{"cancel_attempted_at":null,"keep":true}`), + State: new(rivertype.JobStateRunning), + }), + testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}), + testfactory.Job(ctx, t, exec, &testfactory.JobOpts{FinalizedAt: new(now.Add(-time.Hour)), State: new(rivertype.JobStateCompleted)}), + testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRetryable)}), + } + params := make([]*riverdriver.JobSetStateIfRunningParams, 0, len(original)+1) + for i, job := range original { + params = append(params, riverdriver.JobSetStateCompleted(job.ID, now.Add(time.Duration(i)*time.Second), nil)) + } + params = append(params, riverdriver.JobSetStateCompleted(0, now, nil)) + rows, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams(params...)) + require.NoError(t, err) + require.Len(t, rows, len(original)) + for i, before := range original { + expected := *before + if before.State == rivertype.JobStateRunning { + expected.FinalizedAt = new(now.Add(time.Duration(i) * time.Second)) + expected.State = rivertype.JobStateCompleted + } + require.Equal(t, &expected, rows[i]) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: before.ID}) + require.NoError(t, err) + require.Equal(t, &expected, stored) + } + }) + t.Run("DoesNotCompleteARetryableJob", func(t *testing.T) { t.Parallel() @@ -1197,14 +1233,14 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold a non-array errors value") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, job.ID, "errors", `{"error":"existing value"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job.ID, "errors", `{"error":"existing value"}`) jobsAfter, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams(riverdriver.JobSetStateErrorRetryable(job.ID, now, makeErrPayload(t, now), nil))) require.NoError(t, err) @@ -1278,15 +1314,15 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT t.Parallel() exec, bundle := setup(ctx, t) - if bundle.driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("only SQLite's JSON columns can hold values that don't decode") + if bundle.driver.DatabaseName() == riverdriver.DatabaseNamePostgres { + t.Skip("Postgres uses native array columns") } now := precisionTestTime job1 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) job2 := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) - sqliteSetJobJSONColumn(ctx, t, exec, job2.ID, "tags", `{"not":"an array"}`) + setJobJSONColumn(ctx, t, exec, bundle.driver, "", job2.ID, "tags", `{"not":"an array"}`) jobsAfter, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams( riverdriver.JobSetStateErrorRetryable(job1.ID, now, makeErrPayload(t, now), nil), @@ -1549,6 +1585,75 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT }) }) + t.Run("JobSetStateIfRunningMany_MetadataDoesNotChangeCancellationDecision", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) + cancelled := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Attempt: new(3), Metadata: []byte(`{"cancel_attempted_at":"already requested"}`), + State: new(rivertype.JobStateRunning), + }) + completed := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + FinalizedAt: &precisionTestTime, State: new(rivertype.JobStateCompleted), + }) + rows, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams( + riverdriver.JobSetStateErrorRetryable(job.ID, precisionTestTime, []byte(`{"error":"retry"}`), []byte(`{"cancel_attempted_at":"from metadata update"}`)), + riverdriver.JobSetStateSnoozed(cancelled.ID, precisionTestTime.Add(time.Hour), 2, []byte(`{"cancel_attempted_at":null}`)), + riverdriver.JobSetStateErrorRetryable(completed.ID, precisionTestTime.Add(time.Hour), []byte(`{"error":"ignored"}`), []byte(`{"result":"merged"}`)), + )) + require.NoError(t, err) + require.Len(t, rows, 3) + require.Equal(t, rivertype.JobStateRetryable, rows[0].State) + require.Nil(t, rows[0].FinalizedAt) + require.Equal(t, "from metadata update", gjson.GetBytes(rows[0].Metadata, "cancel_attempted_at").String()) + require.Equal(t, rivertype.JobStateCancelled, rows[1].State) + require.NotNil(t, rows[1].FinalizedAt) + require.Equal(t, cancelled.Attempt, rows[1].Attempt) + require.Equal(t, cancelled.ScheduledAt, rows[1].ScheduledAt) + require.Empty(t, gjson.GetBytes(rows[1].Metadata, "cancel_attempted_at").String()) + require.Equal(t, rivertype.JobStateCompleted, rows[2].State) + require.Equal(t, completed.FinalizedAt, rows[2].FinalizedAt) + require.Equal(t, completed.ScheduledAt, rows[2].ScheduledAt) + require.Empty(t, rows[2].Errors) + require.JSONEq(t, `{"result":"merged"}`, string(rows[2].Metadata)) + }) + + t.Run("JobSetStateIfRunningMany_MetadataMergeIsShallow", func(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"Batch", "RepeatedID", "Single"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + if mode == "RepeatedID" && bundle.driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL retains sequential updates for repeated IDs") + } + now := time.Now().UTC() + job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ + Metadata: []byte(`{"checkpoint":"before","keep":true,"nested":{"old":1}}`), + State: new(rivertype.JobStateRunning), + }) + updates := []*riverdriver.JobSetStateIfRunningParams{riverdriver.JobSetStateCompleted(job.ID, now, []byte(`{"checkpoint":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`))} + switch mode { + case "Batch": + other := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{State: new(rivertype.JobStateRunning)}) + updates = append(updates, riverdriver.JobSetStateCompleted(other.ID, now, []byte(`{"checkpoint":null}`))) + case "RepeatedID": + updates = append(updates, riverdriver.JobSetStateCompleted(job.ID, now, []byte(`{"keep":true}`))) + } + _, err := exec.JobSetStateIfRunningMany(ctx, setStateManyParams(updates...)) + require.NoError(t, err) + stored, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: job.ID}) + require.NoError(t, err) + require.JSONEq(t, `{"checkpoint":null,"keep":true,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`, string(stored.Metadata)) + require.Equal(t, "9007199254740993", gjson.GetBytes(stored.Metadata, "large").Raw) + }) + } + }) + t.Run("JobSetStateIfRunningMany_MultipleJobsAtOnce", func(t *testing.T) { t.Parallel() @@ -1598,16 +1703,16 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT exec, _ := setup(ctx, t) job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{ - Metadata: []byte(`{"key1":"val1"}`), + Metadata: []byte(`{"key1":"val1","nested":{"old":1}}`), }) updatedJob, err := exec.JobUpdate(ctx, &riverdriver.JobUpdateParams{ ID: job.ID, MetadataDoMerge: true, - Metadata: []byte(`{"key2":"val2"}`), + Metadata: []byte(`{"key2":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`), }) require.NoError(t, err) - require.JSONEq(t, `{"key1":"val1","key2":"val2"}`, string(updatedJob.Metadata)) + require.JSONEq(t, `{"key1":"val1","key2":null,"nested":{"new":2},"large":9007199254740993,"a\".b":[true,"text",null]}`, string(updatedJob.Metadata)) }) t.Run("MetadataMergePreservesJSONValuesAndLiteralKeys", func(t *testing.T) { @@ -1624,8 +1729,12 @@ func exerciseJobUpdate[TTx any](ctx context.Context, t *testing.T, executorWithT Metadata: []byte(`{"":false,"a":{"new":true},"a.b":{"new":true},"a\"b.c":[true,false,null],"a\\b":null,"&":true,"Ă©":{},"empty":{},"nullable":null,"large":9007199254740993}`), }) require.NoError(t, err) - require.JSONEq(t, `{"keep":123456789012345678901234567890,"":false,"a":{"new":true},"a.b":{"new":true},"a\"b.c":[true,false,null],"a\\b":null,"&":true,"Ă©":{},"empty":{},"nullable":null,"large":9007199254740993}`, string(updatedJob.Metadata)) - require.Equal(t, "123456789012345678901234567890", gjson.GetBytes(updatedJob.Metadata, "keep").Raw) + // Compare to the stored value: MySQL normalizes numbers outside its + // native integer range when the JSON is first inserted, and rounding + // differs between server versions. + storedNumber := gjson.GetBytes(job.Metadata, "keep").Raw + require.JSONEq(t, fmt.Sprintf(`{"keep":%s,"":false,"a":{"new":true},"a.b":{"new":true},"a\"b.c":[true,false,null],"a\\b":null,"&":true,"Ă©":{},"empty":{},"nullable":null,"large":9007199254740993}`, storedNumber), string(updatedJob.Metadata)) + require.Equal(t, storedNumber, gjson.GetBytes(updatedJob.Metadata, "keep").Raw) require.Equal(t, "9007199254740993", gjson.GetBytes(updatedJob.Metadata, "large").Raw) require.True(t, gjson.GetBytes(updatedJob.Metadata, "a.new").Bool()) }) diff --git a/riverdriver/riverdrivertest/leader.go b/riverdriver/riverdrivertest/leader.go index 7a6cf55f4..57b6283ba 100644 --- a/riverdriver/riverdrivertest/leader.go +++ b/riverdriver/riverdrivertest/leader.go @@ -211,6 +211,29 @@ func exerciseLeader[TTx any](ctx context.Context, t *testing.T, executorWithTx f require.Equal(t, leader.ElectedAt, leaderFromDB.ElectedAt) }) + t.Run("ReelectsWithoutChangingExpiry", func(t *testing.T) { + t.Parallel() + + exec, bundle := setup(ctx, t) + + now := time.Now().UTC().Truncate(bundle.driver.TimePrecision()) + leader, err := exec.LeaderAttemptElect(ctx, &riverdriver.LeaderElectParams{ + LeaderID: testClientID, + Now: &now, + TTL: leaderTTL, + }) + require.NoError(t, err) + + updatedLeader, err := exec.LeaderAttemptReelect(ctx, &riverdriver.LeaderReelectParams{ + ElectedAt: leader.ElectedAt, + LeaderID: testClientID, + Now: &now, + TTL: leaderTTL, + }) + require.NoError(t, err) + require.Equal(t, leader, updatedLeader) + }) + t.Run("WithoutNow", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/riverdrivertest/listener.go b/riverdriver/riverdrivertest/listener.go index 6b8df6de8..9404b16d2 100644 --- a/riverdriver/riverdrivertest/listener.go +++ b/riverdriver/riverdrivertest/listener.go @@ -235,8 +235,8 @@ func exerciseListener[TTx any](ctx context.Context, t *testing.T, driverWithPool listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: ""}) ) - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { - t.Skip("SQLite has no search_path") + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + t.Skipf("%s has no search_path", driver.DatabaseName()) } listener.SetAfterConnectExec("SET search_path TO 'public'") @@ -253,6 +253,10 @@ func exerciseListener[TTx any](ctx context.Context, t *testing.T, driverWithPool listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: ""}) ) + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + t.Skip("MySQL has no search_path") + } + connectListener(ctx, t, listener) require.Empty(t, listener.Schema()) }) diff --git a/riverdriver/riverdrivertest/migration.go b/riverdriver/riverdrivertest/migration.go index bf37e18b1..ec024f1e5 100644 --- a/riverdriver/riverdrivertest/migration.go +++ b/riverdriver/riverdrivertest/migration.go @@ -14,6 +14,7 @@ import ( "github.com/riverqueue/river/rivermigrate" "github.com/riverqueue/river/rivershared/riversharedtest" "github.com/riverqueue/river/rivershared/testfactory" + "github.com/riverqueue/river/rivertype" ) func exerciseMigration[TTx any](ctx context.Context, t *testing.T, @@ -89,6 +90,36 @@ func exerciseMigration[TTx any](ctx context.Context, t *testing.T, require.Contains(t, driver.GetMigrationLines(), riverdriver.MigrationLineMain) }) + t.Run("MigrateTxRejectsMySQLWithoutCommittingCallerWrites", func(t *testing.T) { + t.Parallel() + + driver, schema := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() != riverdriver.DatabaseNameMySQL { + t.Skip("MySQL DDL commits implicitly") + } + migrator, err := rivermigrate.New(driver, &rivermigrate.Config{Schema: schema}) + require.NoError(t, err) + execTx, err := driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = execTx.Rollback(ctx) }) + + _, err = execTx.QueueCreateOrSetUpdatedAt(ctx, &riverdriver.QueueCreateOrSetUpdatedAtParams{ + Metadata: []byte(`{}`), + Name: "migration-rollback-probe", + Schema: schema, + }) + require.NoError(t, err) + _, err = migrator.MigrateTx(ctx, driver.UnwrapTx(execTx), rivermigrate.DirectionDown, &rivermigrate.MigrateOpts{MaxSteps: 2}) //nolint:staticcheck // Verify the deprecated API cannot commit caller writes. + require.ErrorContains(t, err, "MySQL migrations cannot run in a transaction") + require.NoError(t, execTx.Rollback(ctx)) + + _, err = driver.GetExecutor().QueueGet(ctx, &riverdriver.QueueGetParams{Name: "migration-rollback-probe", Schema: schema}) + require.ErrorIs(t, err, rivertype.ErrNotFound) + result, err := migrator.Validate(ctx, nil) + require.NoError(t, err) + require.True(t, result.OK) + }) + // This doesn't map to a particular function, but make sure the driver is // capable of migration all the way up, then all the way back down. t.Run("MigrateUpAndDown", func(t *testing.T) { @@ -253,6 +284,11 @@ func exerciseMigration[TTx any](ctx context.Context, t *testing.T, t.Run("MigrateVersionEightRejectsProSchema", func(t *testing.T) { t.Parallel() + driver, _ := driverWithSchema(ctx, t, nil) + if driver.DatabaseName() != riverdriver.DatabaseNameSQLite { + t.Skip("SQLite table rebuild") + } + for _, testCase := range []struct { name string sql string @@ -279,9 +315,6 @@ func exerciseMigration[TTx any](ctx context.Context, t *testing.T, }, Lines: []string{riverdriver.MigrationLineMain}, }) - if driver.DatabaseName() != riverdriver.DatabaseNameSQLite { - t.Skip("SQLite table rebuild") - } exec := driver.GetExecutor() job := testfactory.Job(ctx, t, exec, &testfactory.JobOpts{Schema: schema}) diff --git a/riverdriver/riverdrivertest/notification.go b/riverdriver/riverdrivertest/notification.go index 2f1025084..fbb547bf5 100644 --- a/riverdriver/riverdrivertest/notification.go +++ b/riverdriver/riverdrivertest/notification.go @@ -31,7 +31,7 @@ func exerciseNotification[TTx any](ctx context.Context, t *testing.T, executorWi ($7, $8, $9), ($10, $11, $12) ` - if driver.DatabaseName() == riverdriver.DatabaseNameSQLite { + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL || driver.DatabaseName() == riverdriver.DatabaseNameSQLite { insertQuery = ` INSERT INTO river_notification (created_at, payload, topic) VALUES diff --git a/riverdriver/riverdrivertest/queue.go b/riverdriver/riverdrivertest/queue.go index 721cf4d61..3c087d483 100644 --- a/riverdriver/riverdrivertest/queue.go +++ b/riverdriver/riverdrivertest/queue.go @@ -77,6 +77,23 @@ func exerciseQueue[TTx any](ctx context.Context, t *testing.T, executorWithTx fu require.WithinDuration(t, now, *queue.PausedAt, bundle.driver.TimePrecision()) }) + t.Run("PreservesExactNames", func(t *testing.T) { + t.Parallel() + + exec, _ := setup(ctx, t) + + for _, name := range []string{"queue", "queue ", "QUEUE", "cafĂ©", "cafe"} { + queue, err := exec.QueueCreateOrSetUpdatedAt(ctx, &riverdriver.QueueCreateOrSetUpdatedAtParams{Name: name}) + require.NoError(t, err) + require.Equal(t, name, queue.Name) + } + for _, name := range []string{"queue", "queue ", "QUEUE", "cafĂ©", "cafe"} { + queue, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{Name: name}) + require.NoError(t, err) + require.Equal(t, name, queue.Name) + } + }) + t.Run("UpdatesTheUpdatedAtOfExistingQueue", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/riverdrivertest/riverdrivertest.go b/riverdriver/riverdrivertest/riverdrivertest.go index b211fe7af..e3f943fc7 100644 --- a/riverdriver/riverdrivertest/riverdrivertest.go +++ b/riverdriver/riverdrivertest/riverdrivertest.go @@ -100,6 +100,25 @@ func exerciseDriverPool[TTx any](ctx context.Context, t *testing.T, }) }) + t.Run("SafeIdentifier", func(t *testing.T) { + t.Parallel() + + _, driver := executorWithTx(ctx, t) + + switch driver.DatabaseName() { + case riverdriver.DatabaseNamePostgres, riverdriver.DatabaseNameSQLite: + require.Equal(t, `"my_schema"`, driver.SafeIdentifier("my_schema")) + require.Equal(t, `"has space"`, driver.SafeIdentifier("has space")) + require.Equal(t, `"has""quote"`, driver.SafeIdentifier(`has"quote`)) + case riverdriver.DatabaseNameMySQL: + require.Equal(t, "`my_schema`", driver.SafeIdentifier("my_schema")) + require.Equal(t, "`has space`", driver.SafeIdentifier("has space")) + require.Equal(t, "`has``backtick`", driver.SafeIdentifier("has`backtick")) + default: + require.FailNow(t, "Don't know how to check SafeIdentifier for: "+driver.DatabaseName()) + } + }) + t.Run("SupportsListenNotify", func(t *testing.T) { t.Parallel() @@ -110,6 +129,8 @@ func exerciseDriverPool[TTx any](ctx context.Context, t *testing.T, require.True(t, driver.SupportsListenNotify()) case riverdriver.DatabaseNameSQLite: require.True(t, driver.SupportsListenNotify()) + case riverdriver.DatabaseNameMySQL: + require.True(t, driver.SupportsListenNotify()) default: require.FailNow(t, "Don't know how to check SupportsListenNotify for: "+driver.DatabaseName()) } @@ -126,7 +147,8 @@ func requireMissingRelation(t *testing.T, err error, schema, missingRelation str // lib/pq: pq: relation %s.%s does not exist // SQLite: no such table: %s.%s // Turso: turso: error: Invalid argument supplied: no such database: %s - require.Regexp(t, fmt.Sprintf(`(pq: relation "%s\.%s" does not exist|no such table: %s\.%s|no such database: %s)`, schema, missingRelation, schema, missingRelation, schema), err.Error()) + // MySQL: Unknown database '%s' + require.Regexp(t, fmt.Sprintf(`(pq: relation "%s\.%s" does not exist|no such table: %s\.%s|no such database: %s|Unknown database '%s')`, schema, missingRelation, schema, missingRelation, schema, schema), err.Error()) } } @@ -140,6 +162,23 @@ var sqliteJobJSONColumns = []string{"args", "attempted_by", "errors", "metadata" // SQLite's JSON functions that touch it. const sqliteMalformedValue = "not json" +// setJobJSONColumn overwrites a JSON column of a SQLite or MySQL job row with the +// given JSON, simulating a row changed out of band into a shape that River +// can't decode. Postgres' column types don't allow the equivalent. +func setJobJSONColumn[TTx any](ctx context.Context, t *testing.T, exec riverdriver.Executor, driver riverdriver.Driver[TTx], schema string, jobID int64, column, jsonValue string) { + t.Helper() + + table := "river_job" + if schema != "" { + table = driver.SafeIdentifier(schema) + "." + table + } + valueSQL := "jsonb(?)" + if driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + valueSQL = "CAST(? AS JSON)" + } + require.NoError(t, exec.Exec(ctx, "UPDATE "+table+" SET "+column+" = "+valueSQL+" WHERE id = ?", jsonValue, jobID)) +} + // sqliteJobColumnText returns a column of a SQLite job row cast to text, which // can be used to check that a value that isn't valid JSON (and so can't be // read through the driver) was left in place. @@ -160,12 +199,3 @@ func sqliteSetJobColumnMalformed(ctx context.Context, t *testing.T, exec riverdr require.NoError(t, exec.Exec(ctx, "UPDATE river_job SET "+column+" = ? WHERE id = ?", sqliteMalformedValue, jobID)) } - -// sqliteSetJobJSONColumn overwrites a JSON column of a SQLite job row with the -// given JSON, simulating a row changed out of band into a shape that River -// can't decode. Postgres' column types don't allow the equivalent. -func sqliteSetJobJSONColumn(ctx context.Context, t *testing.T, exec riverdriver.Executor, jobID int64, column, jsonValue string) { - t.Helper() - - require.NoError(t, exec.Exec(ctx, "UPDATE river_job SET "+column+" = jsonb(?) WHERE id = ?", jsonValue, jobID)) -} diff --git a/riverdriver/riverdrivertest/sql_fragments.go b/riverdriver/riverdrivertest/sql_fragments.go index c0686e4e9..9bce32891 100644 --- a/riverdriver/riverdrivertest/sql_fragments.go +++ b/riverdriver/riverdrivertest/sql_fragments.go @@ -8,6 +8,7 @@ import ( "github.com/riverqueue/river/riverdriver" "github.com/riverqueue/river/rivershared/testfactory" + "github.com/riverqueue/river/rivertype" ) func exerciseSQLFragments[TTx any](ctx context.Context, t *testing.T, executorWithTx func(ctx context.Context, t *testing.T) (riverdriver.Executor, riverdriver.Driver[TTx])) { @@ -95,6 +96,33 @@ func exerciseSQLFragments[TTx any](ctx context.Context, t *testing.T, executorWi require.Equal(t, job2.ID, jobs[1].ID) }) + t.Run("LargeIntegerValues", func(t *testing.T) { + t.Parallel() + + exec, driver := executorWithTx(ctx, t) + + const id = int64(1 << 53) + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: []*riverdriver.JobInsertFastParams{ + {ID: new(id), EncodedArgs: []byte(`{}`), Kind: "large_id", MaxAttempts: 1, Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}}, + {ID: new(id + 1), EncodedArgs: []byte(`{}`), Kind: "large_id", MaxAttempts: 1, Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, Tags: []string{}}, + }}) + require.NoError(t, err) + + for _, filterID := range []int64{id, id + 1} { + fragment, arg, err := driver.SQLFragmentColumnIn("id", []int64{filterID}) + require.NoError(t, err) + jobs, err := exec.JobList(ctx, &riverdriver.JobListParams{ + Max: 100, + NamedArgs: map[string]any{"id": arg}, + OrderByClause: "id", + WhereClause: fragment, + }) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, filterID, jobs[0].ID) + } + }) + t.Run("StringValues", func(t *testing.T) { t.Parallel() diff --git a/riverdriver/rivermysql/example_mysql_test.go b/riverdriver/rivermysql/example_mysql_test.go new file mode 100644 index 000000000..74aa01770 --- /dev/null +++ b/riverdriver/rivermysql/example_mysql_test.go @@ -0,0 +1,125 @@ +package rivermysql_test + +import ( + "cmp" + "context" + "database/sql" + "fmt" + "log/slog" + "os" + "sort" + + _ "github.com/go-sql-driver/mysql" + + "github.com/riverqueue/river" + "github.com/riverqueue/river/riverdriver/rivermysql" + "github.com/riverqueue/river/rivermigrate" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/util/slogutil" + "github.com/riverqueue/river/rivershared/util/testutil" +) + +type MySQLSortArgs struct { + // Strings is a slice of strings to sort. + Strings []string `json:"strings"` +} + +func (MySQLSortArgs) Kind() string { return "sort" } + +type MySQLSortWorker struct { + river.WorkerDefaults[MySQLSortArgs] +} + +func (w *MySQLSortWorker) Work(ctx context.Context, job *river.Job[MySQLSortArgs]) error { + sort.Strings(job.Args.Strings) + fmt.Printf("Sorted strings: %+v\n", job.Args.Strings) + return nil +} + +// Example_mysql demonstrates use of River's MySQL driver. +func Example_mysql() { + // MySQL tests are opt-in because they require a running server. When + // disabled, print the expected output so the example test always passes. + val := os.Getenv("RIVER_MYSQL_TESTS_ENABLED") + if val != "1" && val != "true" { + fmt.Println("Sorted strings: [bear tiger whale]") + return + } + + ctx := context.Background() + + dsn := cmp.Or( + os.Getenv("TEST_MYSQL_URL"), + "root@tcp(localhost:3306)/?parseTime=true&multiStatements=true&loc=UTC&time_zone=%27%2B00%3A00%27", + ) + + dbPool, err := sql.Open("mysql", dsn) + if err != nil { + panic(err) + } + defer dbPool.Close() + + // Create a temporary database for the example. + const exampleDB = "river_example_mysql" + if _, err := dbPool.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+exampleDB); err != nil { + panic(err) + } + defer func() { + _, _ = dbPool.ExecContext(ctx, "DROP DATABASE IF EXISTS "+exampleDB) + }() + + // Run River's migrations to prepare the schema. + migrator, err := rivermigrate.New(rivermysql.New(dbPool), &rivermigrate.Config{ + Logger: slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelWarn, ReplaceAttr: slogutil.NoLevelTime})), + Schema: exampleDB, + }) + if err != nil { + panic(err) + } + if _, err := migrator.Migrate(ctx, rivermigrate.DirectionUp, nil); err != nil { + panic(err) + } + + workers := river.NewWorkers() + river.AddWorker(workers, &MySQLSortWorker{}) + + riverClient, err := river.NewClient(rivermysql.New(dbPool), &river.Config{ + Logger: slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelWarn, ReplaceAttr: slogutil.NoLevelTime})), + Queues: map[string]river.QueueConfig{ + river.QueueDefault: {MaxWorkers: 100}, + }, + Schema: exampleDB, + TestOnly: true, // suitable only for use in tests; remove for live environments + Workers: workers, + }) + if err != nil { + panic(err) + } + + // Out of example scope, but used to wait until a job is worked. + subscribeChan, subscribeCancel := riverClient.Subscribe(river.EventKindJobCompleted) + defer subscribeCancel() + + if err := riverClient.Start(ctx); err != nil { + panic(err) + } + + _, err = riverClient.Insert(ctx, MySQLSortArgs{ + Strings: []string{ + "whale", "tiger", "bear", + }, + }, nil) + if err != nil { + panic(err) + } + + // Wait for jobs to complete. Only needed for purposes of the example test. + riversharedtest.WaitOrTimeoutN(testutil.PanicTB(), subscribeChan, 1) + + if err := riverClient.Stop(ctx); err != nil { + panic(err) + } + + // Output: + // Sorted strings: [bear tiger whale] +} diff --git a/riverdriver/rivermysql/go.mod b/riverdriver/rivermysql/go.mod new file mode 100644 index 000000000..94bf1dba8 --- /dev/null +++ b/riverdriver/rivermysql/go.mod @@ -0,0 +1,31 @@ +module github.com/riverqueue/river/riverdriver/rivermysql + +go 1.26.0 + +toolchain go1.26.6 + +require ( + github.com/go-sql-driver/mysql v1.10.0 + github.com/riverqueue/river v0.51.1 + github.com/riverqueue/river/riverdriver v0.51.1 + github.com/riverqueue/river/rivershared v0.51.1 + github.com/riverqueue/river/rivertype v0.51.1 + github.com/stretchr/testify v1.12.1 + github.com/tidwall/gjson v1.20.0 + github.com/tidwall/sjson v1.2.5 +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.11.0 // indirect + github.com/jackc/puddle/v2 v2.2.3 // indirect + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + go.uber.org/goleak v1.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/text v0.43.0 // indirect +) diff --git a/riverdriver/rivermysql/go.sum b/riverdriver/rivermysql/go.sum new file mode 100644 index 000000000..853d8a5a8 --- /dev/null +++ b/riverdriver/rivermysql/go.sum @@ -0,0 +1,54 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= +github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.20.0 h1:+agJ3rEzKcCXKDKo0ml26UROcpcAlnZyjq+TjbY5Fto= +github.com/tidwall/gjson v1.20.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/riverdriver/rivermysql/internal/dbbatch/job_complete.sql b/riverdriver/rivermysql/internal/dbbatch/job_complete.sql new file mode 100644 index 000000000..71e2a73b0 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_complete.sql @@ -0,0 +1,10 @@ +-- Successful jobs without other updates do not need to materialize old state +-- or inspect cancellation metadata. Completion wins over a cancellation request. +UPDATE /* TEMPLATE: schema */river_job AS job +JOIN JSON_TABLE(?, '$[*]' COLUMNS ( + id BIGINT PATH '$.id' ERROR ON ERROR, + finalized_at DATETIME(6) PATH '$.finalized_at' ERROR ON ERROR +)) AS batch ON batch.id = job.id +SET job.state = 'completed', + job.finalized_at = COALESCE(batch.finalized_at, job.finalized_at) +WHERE job.state = 'running'; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_insert.sql b/riverdriver/rivermysql/internal/dbbatch/job_insert.sql new file mode 100644 index 000000000..ec7d0e458 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_insert.sql @@ -0,0 +1,29 @@ +INSERT INTO /* TEMPLATE: schema */river_job ( + id, args, created_at, kind, max_attempts, metadata, priority, queue, + scheduled_at, state, tags, unique_key, unique_states +) +SELECT + batch.id, CAST(batch.args AS JSON), + COALESCE(batch.created_at, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + batch.kind, batch.max_attempts, CAST(batch.metadata AS JSON), batch.priority, batch.queue, + COALESCE(batch.scheduled_at, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + batch.state, CAST(batch.tags AS JSON), UNHEX(NULLIF(batch.unique_key, '')), NULLIF(batch.unique_states, 0) +FROM JSON_TABLE(?, '$[*]' COLUMNS ( + ordinal FOR ORDINALITY, + id BIGINT PATH '$.i' ERROR ON ERROR, + args LONGTEXT CHARACTER SET utf8mb4 PATH '$.a' ERROR ON ERROR, + created_at DATETIME(6) PATH '$.c' ERROR ON ERROR, + kind LONGTEXT CHARACTER SET utf8mb4 PATH '$.k' ERROR ON ERROR, + max_attempts BIGINT PATH '$.ma' ERROR ON ERROR, + metadata LONGTEXT CHARACTER SET utf8mb4 PATH '$.m' ERROR ON ERROR, + priority BIGINT PATH '$.p' ERROR ON ERROR, + queue LONGTEXT CHARACTER SET utf8mb4 PATH '$.q' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.s' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.st' ERROR ON ERROR, + tags LONGTEXT CHARACTER SET utf8mb4 PATH '$.t' ERROR ON ERROR, + unique_active BOOLEAN PATH '$.u' ERROR ON ERROR, + unique_key LONGTEXT PATH '$.uk' ERROR ON ERROR, + unique_states INT PATH '$.us' ERROR ON ERROR +)) AS batch +ORDER BY batch.ordinal +/* TEMPLATE: on_duplicate_key */; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql b/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql new file mode 100644 index 000000000..2abfbf07c --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_insert_upsert.sql @@ -0,0 +1,20 @@ +-- MySQL cannot target a particular unique index. Only accept a conflict when +-- both rows enforce River uniqueness for the same key. An invalid JSON document +-- makes unrelated conflicts fail atomically, including outside strict SQL mode. +ON DUPLICATE KEY UPDATE id = IF( + batch.unique_active + AND river_job.unique_key = UNHEX(batch.unique_key) + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1, + river_job.id, + CAST('non-River unique constraint conflict' AS JSON) +) diff --git a/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql b/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql new file mode 100644 index 000000000..6333f09e1 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_rescue.sql @@ -0,0 +1,26 @@ +UPDATE /* TEMPLATE: schema */river_job AS job +JOIN JSON_TABLE(?, '$[*]' COLUMNS ( + id BIGINT PATH '$.id' ERROR ON ERROR, + error LONGTEXT CHARACTER SET utf8mb4 PATH '$.error' ERROR ON ERROR, + finalized_at DATETIME(6) PATH '$.finalized_at' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.scheduled_at' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.state' ERROR ON ERROR +)) AS batch ON batch.id = job.id +SET + job.errors = JSON_ARRAY_APPEND(COALESCE(job.errors, JSON_ARRAY()), '$', CAST(batch.error AS JSON)), + job.finalized_at = batch.finalized_at, + job.scheduled_at = batch.scheduled_at, + job.metadata = JSON_SET( + job.metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(job.metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(job.metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(job.metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + job.state = batch.state +WHERE job.state = 'running' AND job.attempted_at < ?; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql b/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql new file mode 100644 index 000000000..5ec11761f --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_set_state.sql @@ -0,0 +1,38 @@ +-- A joined UPDATE doesn't guarantee assignment order. Materialize the old state +-- and cancellation flag before changing either state or metadata. The locking +-- read must see current values even in an existing REPEATABLE READ transaction. +UPDATE /*+ NO_MERGE(batch) */ /* TEMPLATE: schema */river_job AS job +JOIN ( + SELECT input.*, original.state = 'running' AS was_running, + input.state IN ('available', 'retryable', 'scheduled') + AND JSON_CONTAINS_PATH(original.metadata, 'one', '$.cancel_attempted_at') AS should_cancel + FROM JSON_TABLE(?, '$[*]' COLUMNS ( + id BIGINT PATH '$.id' ERROR ON ERROR, + attempt BIGINT PATH '$.attempt' ERROR ON ERROR, + error LONGTEXT CHARACTER SET utf8mb4 PATH '$.error' ERROR ON ERROR, + finalized_at DATETIME(6) PATH '$.finalized_at' ERROR ON ERROR, + metadata LONGTEXT CHARACTER SET utf8mb4 PATH '$.metadata' ERROR ON ERROR, + scheduled_at DATETIME(6) PATH '$.scheduled_at' ERROR ON ERROR, + state LONGTEXT CHARACTER SET utf8mb4 PATH '$.state' ERROR ON ERROR + )) AS input + JOIN /* TEMPLATE: schema */river_job AS original ON original.id = input.id + FOR UPDATE OF original +) AS batch ON batch.id = job.id +SET + job.attempt = CASE WHEN batch.was_running AND NOT batch.should_cancel + THEN COALESCE(batch.attempt, job.attempt) ELSE job.attempt END, + job.errors = CASE WHEN batch.was_running AND batch.error IS NOT NULL + THEN JSON_ARRAY_APPEND(COALESCE(job.errors, JSON_ARRAY()), '$', CAST(batch.error AS JSON)) + ELSE job.errors END, + job.finalized_at = CASE + WHEN batch.was_running AND batch.should_cancel + THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN batch.was_running THEN COALESCE(batch.finalized_at, job.finalized_at) + ELSE job.finalized_at END, + job.scheduled_at = CASE WHEN batch.was_running AND NOT batch.should_cancel + THEN COALESCE(batch.scheduled_at, job.scheduled_at) ELSE job.scheduled_at END, + job.state = CASE + WHEN batch.was_running AND batch.should_cancel THEN 'cancelled' + WHEN batch.was_running THEN batch.state ELSE job.state END, + job.metadata = CASE WHEN batch.metadata IS NOT NULL + THEN CAST(batch.metadata AS JSON) ELSE job.metadata END; diff --git a/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql b/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql new file mode 100644 index 000000000..e5d042ec8 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/job_unique_key.sql @@ -0,0 +1,16 @@ +-- Match the stored index expression, including its explicit string charset. +CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN _utf8mb4'available' THEN unique_states & (1 << 0) + WHEN _utf8mb4'cancelled' THEN unique_states & (1 << 1) + WHEN _utf8mb4'completed' THEN unique_states & (1 << 2) + WHEN _utf8mb4'discarded' THEN unique_states & (1 << 3) + WHEN _utf8mb4'pending' THEN unique_states & (1 << 4) + WHEN _utf8mb4'retryable' THEN unique_states & (1 << 5) + WHEN _utf8mb4'running' THEN unique_states & (1 << 6) + WHEN _utf8mb4'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END diff --git a/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql b/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql new file mode 100644 index 000000000..b17100b4a --- /dev/null +++ b/riverdriver/rivermysql/internal/dbbatch/notification_insert.sql @@ -0,0 +1,8 @@ +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +SELECT batch.payload, config.topic +FROM JSON_TABLE(?, '$[*]' COLUMNS ( + ordinal FOR ORDINALITY, + payload LONGTEXT CHARACTER SET utf8mb4 PATH '$' ERROR ON ERROR +)) AS batch +CROSS JOIN (SELECT ? AS topic) AS config +ORDER BY batch.ordinal; diff --git a/riverdriver/rivermysql/internal/dbfragment/json_contains.sql b/riverdriver/rivermysql/internal/dbfragment/json_contains.sql new file mode 100644 index 000000000..da4dda2d4 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbfragment/json_contains.sql @@ -0,0 +1,5 @@ +{{define "container"}}(JSON_TYPE({{.source}}) = '{{.kind}}' AND {{.children}}){{end}} +{{define "member"}}(SELECT 1 FROM JSON_TABLE(JSON_EXTRACT({{.source}}, {{.param}}), '$' COLUMNS(value JSON PATH '$')) AS {{.alias}} WHERE {{.predicate}} LIMIT 1) IS NOT NULL{{end}} +{{define "element"}}(SELECT 1 FROM JSON_TABLE({{.source}}, '$[*]' COLUMNS(value JSON PATH '$')) AS {{.alias}} WHERE {{.predicate}} LIMIT 1) IS NOT NULL{{end}} +{{define "null"}}({{.source}} IS NULL OR JSON_TYPE({{.source}}) = 'NULL'){{end}} +{{define "scalar"}}(JSON_TYPE({{.source}}) NOT IN ('ARRAY', 'OBJECT') AND JSON_CONTAINS({{.source}}, CAST({{.param}} AS JSON))){{end}} diff --git a/riverdriver/rivermysql/internal/dbsqlc/db.go b/riverdriver/rivermysql/internal/dbsqlc/db.go new file mode 100644 index 000000000..c399d01ff --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/db.go @@ -0,0 +1,24 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package dbsqlc + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New() *Queries { + return &Queries{} +} + +type Queries struct { +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/models.go b/riverdriver/rivermysql/internal/dbsqlc/models.go new file mode 100644 index 000000000..7aa14da32 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/models.go @@ -0,0 +1,82 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package dbsqlc + +import ( + "database/sql" + "time" + + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbtype" +) + +type Columns struct { + ColumnName string + TableName string + TableSchema string +} + +type RiverJob struct { + ID int64 + Args dbtype.JSON + Attempt int64 + AttemptedAt sql.NullTime + AttemptedBy dbtype.JSON + CreatedAt time.Time + Errors dbtype.JSON + FinalizedAt sql.NullTime + Kind string + MaxAttempts int64 + Metadata dbtype.JSON + Priority int16 + Queue string + State string + ScheduledAt time.Time + Tags dbtype.JSON + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +type RiverLeader struct { + ElectedAt time.Time + ExpiresAt time.Time + LeaderID string + Name string +} + +type RiverMigration struct { + Line string + Version int64 + CreatedAt time.Time +} + +type RiverNotification struct { + ID int64 + CreatedAt time.Time + Payload string + Topic string +} + +type RiverQueue struct { + Name string + CreatedAt time.Time + Metadata dbtype.JSON + PausedAt sql.NullTime + UpdatedAt time.Time +} + +type Schemata struct { + SchemaName string +} + +type Statistics struct { + IndexName string + TableName string + TableSchema string +} + +type Tables struct { + TableName string + TableSchema string +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_job.sql b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql new file mode 100644 index 000000000..91a127f0a --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql @@ -0,0 +1,487 @@ +CREATE TABLE river_job ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + args JSON NOT NULL DEFAULT (JSON_OBJECT()), + attempt INT NOT NULL DEFAULT 0, + attempted_at DATETIME(6) NULL, + attempted_by JSON NULL, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + errors JSON NULL, + finalized_at DATETIME(6) NULL, + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + max_attempts INT NOT NULL DEFAULT 25, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + priority SMALLINT NOT NULL DEFAULT 1, + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + scheduled_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + tags JSON NOT NULL DEFAULT (JSON_ARRAY()), + unique_key VARBINARY(255) NULL, + unique_states SMALLINT NULL +); + +-- name: JobGetByID :one +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') +LIMIT 1; + +-- name: JobGetByIDMany :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id; + +-- name: JobGetByIDManyForUpdate :many +-- Read current versions after writes, even in a pre-existing REPEATABLE READ transaction. +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id +FOR UPDATE; + +-- name: JobGetByInsertNonce :many +-- Own writes are visible even in an older REPEATABLE READ snapshot. Do not use +-- a locking read: the range can contain interleaved inserts from other sessions. +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id >= sqlc.arg('first_id') + AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$."river:unique_nonce"')) LIKE sqlc.arg('nonce_prefix') +ORDER BY id; + +-- name: JobGetByKindMany :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE kind IN (sqlc.slice('kind')) +ORDER BY id; + +-- name: JobGetByUniqueKeyManyForUpdate :many +-- Match the expression in river_job_unique_idx so lookups use the unique index. +-- A current read is needed for existing jobs committed after a caller's snapshot. +-- Force the index to avoid locking unrelated rows if a table scan looks cheaper. +SELECT * +FROM /* TEMPLATE: schema */river_job FORCE INDEX (river_job_unique_idx) +WHERE /* TEMPLATE_BEGIN: unique_key */ unique_key /* TEMPLATE_END */ IN (sqlc.slice('unique_keys')) +ORDER BY id +FOR UPDATE; + +-- name: JobCancelExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END, + finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) END, + metadata = JSON_SET(metadata, '$.cancel_attempted_at', CAST(sqlc.arg('cancel_attempted_at') AS CHAR)) +WHERE id = sqlc.arg('id') + AND state NOT IN ('cancelled', 'completed', 'discarded') + AND finalized_at IS NULL; + +-- name: JobClearUniqueNonce :exec +-- Avoid a table scan locking other sessions' uncommitted inserts. +UPDATE /* TEMPLATE: schema */river_job FORCE INDEX (PRIMARY) +SET metadata = JSON_REMOVE(metadata, '$."river:unique_nonce"') +WHERE id IN (sqlc.slice('id')); + +-- name: JobCountByAllStates :many +SELECT state, count(*) AS count +FROM /* TEMPLATE: schema */river_job +GROUP BY state; + +-- name: JobCountByQueueAndState :many +SELECT + queue, + COUNT(CASE WHEN state = 'available' THEN 1 END) AS count_available, + COUNT(CASE WHEN state = 'running' THEN 1 END) AS count_running +FROM /* TEMPLATE: schema */river_job +WHERE queue IN (sqlc.slice('queue_names')) + AND state IN ('available', 'running') +GROUP BY queue +ORDER BY queue; + +-- name: JobCountByState :one +SELECT count(*) AS count +FROM /* TEMPLATE: schema */river_job +WHERE state = sqlc.arg('state'); + +-- name: JobDeleteExec :execresult +DELETE FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') + AND river_job.state != 'running'; + +-- name: JobGetMetadataForUpdate :many +SELECT /*+ JOIN_INDEX(river_job PRIMARY) */ id, metadata +FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) +ORDER BY id +FOR UPDATE; + +-- name: JobGetByIDForUpdate :one +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE id = sqlc.arg('id') +LIMIT 1 +FOR UPDATE; + +-- name: JobDeleteBeforeSelect :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE ( + ( + CAST(sqlc.arg('cancelled_do_delete') AS UNSIGNED) + AND state = 'cancelled' + AND finalized_at < sqlc.arg('cancelled_finalized_at_horizon') + ) OR ( + CAST(sqlc.arg('completed_do_delete') AS UNSIGNED) + AND state = 'completed' + AND finalized_at < sqlc.arg('completed_finalized_at_horizon') + ) OR ( + CAST(sqlc.arg('discarded_do_delete') AS UNSIGNED) + AND state = 'discarded' + AND finalized_at < sqlc.arg('discarded_finalized_at_horizon') + ) +) +AND ( + CAST(sqlc.arg('queues_excluded_empty') AS SIGNED) + OR queue NOT IN (sqlc.slice('queues_excluded')) +) +AND ( + CAST(sqlc.arg('queues_included_empty') AS SIGNED) + OR queue IN (sqlc.slice('queues_included')) +) +ORDER BY id +LIMIT ? +FOR UPDATE SKIP LOCKED; + +-- name: JobDeleteManySelect :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + AND state != 'running' +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +FOR UPDATE; + +-- Keep large batches on primary-key ranges so a scan cannot lock skipped jobs. +-- name: JobDeleteManyExec :exec +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')); + +-- name: JobGetAvailable :many +-- Project the updated attempt fields while acquiring row locks. The executor +-- supplies the running state. Keep this projection and JobGetAvailableUpdate +-- in sync; both use the same timestamp and attempted_by expression. +SELECT id, args, CAST(attempt + 1 AS SIGNED) AS attempt, + CAST(COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) AS DATETIME(6)) AS attempted_at, + JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(sqlc.arg('max_attempted_by') AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(sqlc.arg('max_attempted_by') AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(sqlc.arg('max_attempted_by') AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(sqlc.arg('attempted_by') AS CHAR) + ) AS attempted_by, + created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, + state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE + priority >= 0 + AND queue = sqlc.arg('queue') + AND scheduled_at <= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND state = 'available' + -- The first comparison permits use of the kind index; the binary + -- comparison preserves exact matching under PAD SPACE collations. + AND (sqlc.arg('kind_filter') = 0 OR ( + kind IN (sqlc.slice('kind')) + AND /* TEMPLATE_BEGIN: kind_column */ kind /* TEMPLATE_END */ IN (sqlc.slice('kind')) + )) +ORDER BY priority ASC, scheduled_at ASC, id ASC +LIMIT ? +FOR UPDATE SKIP LOCKED; + +-- name: JobGetAvailableUpdate :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = attempt + 1, + attempted_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + attempted_by = JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(sqlc.arg('max_attempted_by') AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(sqlc.arg('max_attempted_by') AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(sqlc.arg('max_attempted_by') AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(sqlc.arg('attempted_by') AS CHAR) + ), + state = 'running' +WHERE id IN (sqlc.slice('id')); + +-- name: JobGetCancelRequested :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE id IN (sqlc.slice('id')) + AND state = 'running' + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') +ORDER BY id; + +-- name: JobGetStuck :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE state = 'running' + AND id > sqlc.arg('after_id') + AND attempted_at < sqlc.arg('stuck_horizon') +ORDER BY id +LIMIT ?; + +-- name: JobInsertFast :execresult +INSERT INTO /* TEMPLATE: schema */river_job( + id, + args, + created_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + sqlc.narg('id'), + sqlc.arg('args'), + COALESCE(sqlc.narg('created_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('kind'), + sqlc.arg('max_attempts'), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('priority'), + sqlc.arg('queue'), + COALESCE(sqlc.narg('scheduled_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('state'), + CAST(sqlc.arg('tags') AS JSON), + sqlc.narg('unique_key'), + sqlc.narg('unique_states') +) /* TEMPLATE_BEGIN: on_duplicate_key */ ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) /* TEMPLATE_END */; + +-- name: JobInsertFullExec :execlastid +INSERT INTO /* TEMPLATE: schema */river_job( + args, + attempt, + attempted_at, + attempted_by, + created_at, + errors, + finalized_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + sqlc.arg('args'), + sqlc.arg('attempt'), + sqlc.narg('attempted_at'), + CAST(sqlc.narg('attempted_by') AS JSON), + COALESCE(sqlc.narg('created_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(sqlc.narg('errors') AS JSON), + sqlc.narg('finalized_at'), + sqlc.arg('kind'), + sqlc.arg('max_attempts'), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('priority'), + sqlc.arg('queue'), + COALESCE(sqlc.narg('scheduled_at'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + sqlc.arg('state'), + CAST(sqlc.arg('tags') AS JSON), + sqlc.narg('unique_key'), + sqlc.narg('unique_states') +); + +-- name: JobKindList :many +SELECT DISTINCT kind +FROM /* TEMPLATE: schema */river_job +WHERE (sqlc.arg('match') = '' OR LOWER(kind) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(sqlc.arg('match')), '%')) + AND (sqlc.arg('after') = '' OR kind > sqlc.arg('after')) + AND kind NOT IN (sqlc.slice('exclude')) +ORDER BY kind ASC +LIMIT ?; + +-- name: JobList :many +SELECT * +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ?; + +-- name: JobRescue :exec +UPDATE /* TEMPLATE: schema */river_job +SET + errors = JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(sqlc.arg('error') AS JSON)), + finalized_at = sqlc.narg('finalized_at'), + scheduled_at = sqlc.arg('scheduled_at'), + metadata = JSON_SET( + metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + state = sqlc.arg('state') +WHERE id = sqlc.arg('id') + AND state = 'running' + AND attempted_at < sqlc.arg('stuck_horizon'); + +-- name: JobRetryExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = 'available', + max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END, + finalized_at = NULL, + metadata = JSON_REMOVE(metadata, '$.cancel_attempted_at'), + scheduled_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +WHERE id = sqlc.arg('id') + AND state != 'running' + AND ( + state <> 'available' + OR scheduled_at > COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ); + +-- name: JobSchedule :many +WITH eligible AS ( + SELECT river_job.id, river_job.unique_key, river_job.unique_states, river_job.priority, river_job.scheduled_at, + CASE + WHEN river_job.unique_key IS NOT NULL AND river_job.unique_states IS NOT NULL THEN + ROW_NUMBER() OVER (PARTITION BY river_job.unique_key ORDER BY river_job.priority, river_job.scheduled_at, river_job.id) + ELSE NULL + END AS row_num + FROM /* TEMPLATE: schema */river_job + WHERE + river_job.state IN ('retryable', 'scheduled') + AND river_job.scheduled_at <= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ORDER BY + river_job.priority, + river_job.scheduled_at, + river_job.id + LIMIT ? + FOR UPDATE SKIP LOCKED +), +unique_conflicts AS ( + SELECT DISTINCT eligible.unique_key + FROM /* TEMPLATE: schema */river_job + JOIN eligible + ON river_job.unique_key = eligible.unique_key + AND river_job.id != eligible.id + WHERE + river_job.unique_key IS NOT NULL + AND river_job.unique_states IS NOT NULL + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1 +) +SELECT eligible.id, + CASE + WHEN eligible.unique_key IS NULL OR eligible.unique_states IS NULL THEN FALSE + WHEN uc.unique_key IS NOT NULL THEN TRUE + WHEN eligible.row_num > 1 THEN TRUE + ELSE FALSE + END AS conflict_discarded +FROM eligible +LEFT JOIN unique_conflicts uc ON eligible.unique_key = uc.unique_key +ORDER BY eligible.priority, eligible.scheduled_at, eligible.id; + +-- name: JobScheduleSetAvailableExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET state = 'available' +WHERE id IN (sqlc.slice('id')); + +-- name: JobScheduleSetDiscardedExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET metadata = JSON_MERGE_PATCH(metadata, '{"unique_key_conflict": "scheduler_discarded"}'), + finalized_at = COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + state = 'discarded' +WHERE id IN (sqlc.slice('id')); + +-- name: JobSetStateIfRunningExec :exec +-- MySQL evaluates assignments left-to-right. Keep state after the fields whose +-- updates depend on it, and metadata last so cancellation checks use the old metadata. +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE + WHEN state = 'running' + AND ((CAST(sqlc.arg('state') AS CHAR) <> 'available' AND sqlc.arg('state') <> 'retryable' AND sqlc.arg('state') <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(sqlc.arg('attempt_do_update') AS SIGNED) + THEN sqlc.arg('attempt') ELSE attempt END, + errors = CASE + WHEN state = 'running' AND CAST(sqlc.arg('errors_do_update') AS SIGNED) + THEN JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(sqlc.arg('error') AS JSON)) + ELSE errors END, + finalized_at = CASE + WHEN state = 'running' AND ((sqlc.arg('state') = 'available' OR sqlc.arg('state') = 'retryable' OR sqlc.arg('state') = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN state = 'running' AND CAST(sqlc.arg('finalized_at_do_update') AS SIGNED) + THEN sqlc.narg('finalized_at') ELSE finalized_at END, + scheduled_at = CASE + WHEN state = 'running' + AND ((sqlc.arg('state') <> 'available' AND sqlc.arg('state') <> 'retryable' AND sqlc.arg('state') <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(sqlc.arg('scheduled_at_do_update') AS SIGNED) + THEN sqlc.arg('scheduled_at') ELSE scheduled_at END, + state = CASE + WHEN state = 'running' AND ((sqlc.arg('state') = 'available' OR sqlc.arg('state') = 'retryable' OR sqlc.arg('state') = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN 'cancelled' + WHEN state = 'running' THEN sqlc.arg('state') ELSE state END, + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_merge') AS SIGNED) + THEN CAST(sqlc.arg('metadata_updates') AS JSON) ELSE metadata END +WHERE id = sqlc.arg('id'); + +-- name: JobUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_merge') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END +WHERE id = sqlc.arg('id'); + +-- name: JobUpdateFullExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE WHEN CAST(sqlc.arg('attempt_do_update') AS SIGNED) THEN sqlc.arg('attempt') ELSE attempt END, + attempted_at = CASE WHEN CAST(sqlc.arg('attempted_at_do_update') AS SIGNED) THEN sqlc.narg('attempted_at') ELSE attempted_at END, + attempted_by = CASE WHEN CAST(sqlc.arg('attempted_by_do_update') AS SIGNED) THEN CAST(sqlc.arg('attempted_by') AS JSON) ELSE attempted_by END, + errors = CASE WHEN CAST(sqlc.arg('errors_do_update') AS SIGNED) THEN CAST(sqlc.arg('errors') AS JSON) ELSE errors END, + finalized_at = CASE WHEN CAST(sqlc.arg('finalized_at_do_update') AS SIGNED) THEN sqlc.narg('finalized_at') ELSE finalized_at END, + max_attempts = CASE WHEN CAST(sqlc.arg('max_attempts_do_update') AS SIGNED) THEN sqlc.arg('max_attempts') ELSE max_attempts END, + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_update') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END, + state = CASE WHEN CAST(sqlc.arg('state_do_update') AS SIGNED) THEN sqlc.arg('state') ELSE state END +WHERE id = sqlc.arg('id'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go new file mode 100644 index 000000000..7a5c89440 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_job.sql.go @@ -0,0 +1,1656 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_job.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" + "time" + + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbtype" +) + +const jobCancelExec = `-- name: JobCancelExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END, + finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) END, + metadata = JSON_SET(metadata, '$.cancel_attempted_at', CAST(? AS CHAR)) +WHERE id = ? + AND state NOT IN ('cancelled', 'completed', 'discarded') + AND finalized_at IS NULL +` + +type JobCancelExecParams struct { + Now sql.NullTime + CancelAttemptedAt interface{} + ID int64 +} + +func (q *Queries) JobCancelExec(ctx context.Context, db DBTX, arg *JobCancelExecParams) (sql.Result, error) { + return db.ExecContext(ctx, jobCancelExec, arg.Now, arg.CancelAttemptedAt, arg.ID) +} + +const jobClearUniqueNonce = `-- name: JobClearUniqueNonce :exec +UPDATE /* TEMPLATE: schema */river_job FORCE INDEX (PRIMARY) +SET metadata = JSON_REMOVE(metadata, '$."river:unique_nonce"') +WHERE id IN (/*SLICE:id*/?) +` + +// Avoid a table scan locking other sessions' uncommitted inserts. +func (q *Queries) JobClearUniqueNonce(ctx context.Context, db DBTX, id []int64) error { + query := jobClearUniqueNonce + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobCountByAllStates = `-- name: JobCountByAllStates :many +SELECT state, count(*) AS count +FROM /* TEMPLATE: schema */river_job +GROUP BY state +` + +type JobCountByAllStatesRow struct { + State string + Count int64 +} + +func (q *Queries) JobCountByAllStates(ctx context.Context, db DBTX) ([]*JobCountByAllStatesRow, error) { + rows, err := db.QueryContext(ctx, jobCountByAllStates) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobCountByAllStatesRow + for rows.Next() { + var i JobCountByAllStatesRow + if err := rows.Scan(&i.State, &i.Count); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobCountByQueueAndState = `-- name: JobCountByQueueAndState :many +SELECT + queue, + COUNT(CASE WHEN state = 'available' THEN 1 END) AS count_available, + COUNT(CASE WHEN state = 'running' THEN 1 END) AS count_running +FROM /* TEMPLATE: schema */river_job +WHERE queue IN (/*SLICE:queue_names*/?) + AND state IN ('available', 'running') +GROUP BY queue +ORDER BY queue +` + +type JobCountByQueueAndStateRow struct { + Queue string + CountAvailable int64 + CountRunning int64 +} + +func (q *Queries) JobCountByQueueAndState(ctx context.Context, db DBTX, queueNames []string) ([]*JobCountByQueueAndStateRow, error) { + query := jobCountByQueueAndState + var queryParams []interface{} + if len(queueNames) > 0 { + for _, v := range queueNames { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queue_names*/?", strings.Repeat(",?", len(queueNames))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queue_names*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobCountByQueueAndStateRow + for rows.Next() { + var i JobCountByQueueAndStateRow + if err := rows.Scan(&i.Queue, &i.CountAvailable, &i.CountRunning); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobCountByState = `-- name: JobCountByState :one +SELECT count(*) AS count +FROM /* TEMPLATE: schema */river_job +WHERE state = ? +` + +func (q *Queries) JobCountByState(ctx context.Context, db DBTX, state string) (int64, error) { + row := db.QueryRowContext(ctx, jobCountByState, state) + var count int64 + err := row.Scan(&count) + return count, err +} + +const jobDeleteBeforeSelect = `-- name: JobDeleteBeforeSelect :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE ( + ( + CAST(? AS UNSIGNED) + AND state = 'cancelled' + AND finalized_at < ? + ) OR ( + CAST(? AS UNSIGNED) + AND state = 'completed' + AND finalized_at < ? + ) OR ( + CAST(? AS UNSIGNED) + AND state = 'discarded' + AND finalized_at < ? + ) +) +AND ( + CAST(? AS SIGNED) + OR queue NOT IN (/*SLICE:queues_excluded*/?) +) +AND ( + CAST(? AS SIGNED) + OR queue IN (/*SLICE:queues_included*/?) +) +ORDER BY id +LIMIT ? +FOR UPDATE SKIP LOCKED +` + +type JobDeleteBeforeSelectParams struct { + CancelledDoDelete int64 + CancelledFinalizedAtHorizon sql.NullTime + CompletedDoDelete int64 + CompletedFinalizedAtHorizon sql.NullTime + DiscardedDoDelete int64 + DiscardedFinalizedAtHorizon sql.NullTime + QueuesExcludedEmpty int64 + QueuesExcluded []string + QueuesIncludedEmpty int64 + QueuesIncluded []string + Limit int32 +} + +func (q *Queries) JobDeleteBeforeSelect(ctx context.Context, db DBTX, arg *JobDeleteBeforeSelectParams) ([]int64, error) { + query := jobDeleteBeforeSelect + var queryParams []interface{} + queryParams = append(queryParams, arg.CancelledDoDelete) + queryParams = append(queryParams, arg.CancelledFinalizedAtHorizon) + queryParams = append(queryParams, arg.CompletedDoDelete) + queryParams = append(queryParams, arg.CompletedFinalizedAtHorizon) + queryParams = append(queryParams, arg.DiscardedDoDelete) + queryParams = append(queryParams, arg.DiscardedFinalizedAtHorizon) + queryParams = append(queryParams, arg.QueuesExcludedEmpty) + if len(arg.QueuesExcluded) > 0 { + for _, v := range arg.QueuesExcluded { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queues_excluded*/?", strings.Repeat(",?", len(arg.QueuesExcluded))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queues_excluded*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.QueuesIncludedEmpty) + if len(arg.QueuesIncluded) > 0 { + for _, v := range arg.QueuesIncluded { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:queues_included*/?", strings.Repeat(",?", len(arg.QueuesIncluded))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:queues_included*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobDeleteExec = `-- name: JobDeleteExec :execresult +DELETE FROM /* TEMPLATE: schema */river_job +WHERE id = ? + AND river_job.state != 'running' +` + +func (q *Queries) JobDeleteExec(ctx context.Context, db DBTX, id int64) (sql.Result, error) { + return db.ExecContext(ctx, jobDeleteExec, id) +} + +const jobDeleteManyExec = `-- name: JobDeleteManyExec :exec +DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +` + +// Keep large batches on primary-key ranges so a scan cannot lock skipped jobs. +func (q *Queries) JobDeleteManyExec(ctx context.Context, db DBTX, id []int64) error { + query := jobDeleteManyExec + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobDeleteManySelect = `-- name: JobDeleteManySelect :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + AND state != 'running' +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +FOR UPDATE +` + +func (q *Queries) JobDeleteManySelect(ctx context.Context, db DBTX, limit int32) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobDeleteManySelect, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetAvailable = `-- name: JobGetAvailable :many +SELECT id, args, CAST(attempt + 1 AS SIGNED) AS attempt, + CAST(COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) AS DATETIME(6)) AS attempted_at, + JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(? AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(? AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(? AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(? AS CHAR) + ) AS attempted_by, + created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, + state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE + priority >= 0 + AND queue = ? + AND scheduled_at <= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND state = 'available' + -- The first comparison permits use of the kind index; the binary + -- comparison preserves exact matching under PAD SPACE collations. + AND (? = 0 OR ( + kind IN (/*SLICE:kind*/?) + AND /* TEMPLATE_BEGIN: kind_column */ kind /* TEMPLATE_END */ IN (/*SLICE:kind*/?) + )) +ORDER BY priority ASC, scheduled_at ASC, id ASC +LIMIT ? +FOR UPDATE SKIP LOCKED +` + +type JobGetAvailableParams struct { + Now sql.NullTime + MaxAttemptedBy int64 + AttemptedBy interface{} + Queue string + KindFilter interface{} + Kind []string + Limit int32 +} + +type JobGetAvailableRow struct { + ID int64 + Args dbtype.JSON + Attempt int64 + AttemptedAt time.Time + AttemptedBy dbtype.JSON + CreatedAt time.Time + Errors dbtype.JSON + FinalizedAt sql.NullTime + Kind string + MaxAttempts int64 + Metadata dbtype.JSON + Priority int16 + Queue string + State string + ScheduledAt time.Time + Tags dbtype.JSON + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +// Project the updated attempt fields while acquiring row locks. The executor +// supplies the running state. Keep this projection and JobGetAvailableUpdate +// in sync; both use the same timestamp and attempted_by expression. +func (q *Queries) JobGetAvailable(ctx context.Context, db DBTX, arg *JobGetAvailableParams) ([]*JobGetAvailableRow, error) { + query := jobGetAvailable + var queryParams []interface{} + queryParams = append(queryParams, arg.Now) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.AttemptedBy) + queryParams = append(queryParams, arg.Queue) + queryParams = append(queryParams, arg.Now) + queryParams = append(queryParams, arg.KindFilter) + if len(arg.Kind) > 0 { + for _, v := range arg.Kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(arg.Kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + if len(arg.Kind) > 0 { + for _, v := range arg.Kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(arg.Kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobGetAvailableRow + for rows.Next() { + var i JobGetAvailableRow + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetAvailableUpdate = `-- name: JobGetAvailableUpdate :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = attempt + 1, + attempted_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + attempted_by = JSON_ARRAY_APPEND( + CASE + WHEN JSON_LENGTH(COALESCE(attempted_by, JSON_ARRAY())) < CAST(? AS SIGNED) + THEN COALESCE(attempted_by, JSON_ARRAY()) + WHEN CAST(? AS SIGNED) <= 1 + THEN JSON_ARRAY() + ELSE COALESCE( + JSON_EXTRACT( + attempted_by, + CONCAT('$[last-', CAST(? AS SIGNED) - 2, ' to last]') + ), + JSON_ARRAY() + ) + END, + '$', + CAST(? AS CHAR) + ), + state = 'running' +WHERE id IN (/*SLICE:id*/?) +` + +type JobGetAvailableUpdateParams struct { + Now sql.NullTime + MaxAttemptedBy int64 + AttemptedBy interface{} + ID []int64 +} + +func (q *Queries) JobGetAvailableUpdate(ctx context.Context, db DBTX, arg *JobGetAvailableUpdateParams) error { + query := jobGetAvailableUpdate + var queryParams []interface{} + queryParams = append(queryParams, arg.Now) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.MaxAttemptedBy) + queryParams = append(queryParams, arg.AttemptedBy) + if len(arg.ID) > 0 { + for _, v := range arg.ID { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(arg.ID))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobGetByID = `-- name: JobGetByID :one +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id = ? +LIMIT 1 +` + +func (q *Queries) JobGetByID(ctx context.Context, db DBTX, id int64) (*RiverJob, error) { + row := db.QueryRowContext(ctx, jobGetByID, id) + var i RiverJob + err := row.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ) + return &i, err +} + +const jobGetByIDForUpdate = `-- name: JobGetByIDForUpdate :one +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id = ? +LIMIT 1 +FOR UPDATE +` + +func (q *Queries) JobGetByIDForUpdate(ctx context.Context, db DBTX, id int64) (*RiverJob, error) { + row := db.QueryRowContext(ctx, jobGetByIDForUpdate, id) + var i RiverJob + err := row.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ) + return &i, err +} + +const jobGetByIDMany = `-- name: JobGetByIDMany :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +` + +func (q *Queries) JobGetByIDMany(ctx context.Context, db DBTX, id []int64) ([]*RiverJob, error) { + query := jobGetByIDMany + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByIDManyForUpdate = `-- name: JobGetByIDManyForUpdate :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +FOR UPDATE +` + +// Read current versions after writes, even in a pre-existing REPEATABLE READ transaction. +func (q *Queries) JobGetByIDManyForUpdate(ctx context.Context, db DBTX, id []int64) ([]*RiverJob, error) { + query := jobGetByIDManyForUpdate + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByInsertNonce = `-- name: JobGetByInsertNonce :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE id >= ? + AND JSON_UNQUOTE(JSON_EXTRACT(metadata, '$."river:unique_nonce"')) LIKE ? +ORDER BY id +` + +type JobGetByInsertNonceParams struct { + FirstID int64 + NoncePrefix dbtype.JSON +} + +// Own writes are visible even in an older REPEATABLE READ snapshot. Do not use +// a locking read: the range can contain interleaved inserts from other sessions. +func (q *Queries) JobGetByInsertNonce(ctx context.Context, db DBTX, arg *JobGetByInsertNonceParams) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobGetByInsertNonce, arg.FirstID, arg.NoncePrefix) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByKindMany = `-- name: JobGetByKindMany :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE kind IN (/*SLICE:kind*/?) +ORDER BY id +` + +func (q *Queries) JobGetByKindMany(ctx context.Context, db DBTX, kind []string) ([]*RiverJob, error) { + query := jobGetByKindMany + var queryParams []interface{} + if len(kind) > 0 { + for _, v := range kind { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:kind*/?", strings.Repeat(",?", len(kind))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:kind*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetByUniqueKeyManyForUpdate = `-- name: JobGetByUniqueKeyManyForUpdate :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job FORCE INDEX (river_job_unique_idx) +WHERE /* TEMPLATE_BEGIN: unique_key */ unique_key /* TEMPLATE_END */ IN (/*SLICE:unique_keys*/?) +ORDER BY id +FOR UPDATE +` + +// Match the expression in river_job_unique_idx so lookups use the unique index. +// A current read is needed for existing jobs committed after a caller's snapshot. +// Force the index to avoid locking unrelated rows if a table scan looks cheaper. +func (q *Queries) JobGetByUniqueKeyManyForUpdate(ctx context.Context, db DBTX, uniqueKeys []sql.NullString) ([]*RiverJob, error) { + query := jobGetByUniqueKeyManyForUpdate + var queryParams []interface{} + if len(uniqueKeys) > 0 { + for _, v := range uniqueKeys { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:unique_keys*/?", strings.Repeat(",?", len(uniqueKeys))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:unique_keys*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetCancelRequested = `-- name: JobGetCancelRequested :many +SELECT id FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) + AND state = 'running' + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') +ORDER BY id +` + +func (q *Queries) JobGetCancelRequested(ctx context.Context, db DBTX, id []int64) ([]int64, error) { + query := jobGetCancelRequested + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetMetadataForUpdate = `-- name: JobGetMetadataForUpdate :many +SELECT /*+ JOIN_INDEX(river_job PRIMARY) */ id, metadata +FROM /* TEMPLATE: schema */river_job +WHERE id IN (/*SLICE:id*/?) +ORDER BY id +FOR UPDATE +` + +type JobGetMetadataForUpdateRow struct { + ID int64 + Metadata dbtype.JSON +} + +func (q *Queries) JobGetMetadataForUpdate(ctx context.Context, db DBTX, id []int64) ([]*JobGetMetadataForUpdateRow, error) { + query := jobGetMetadataForUpdate + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobGetMetadataForUpdateRow + for rows.Next() { + var i JobGetMetadataForUpdateRow + if err := rows.Scan(&i.ID, &i.Metadata); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobGetStuck = `-- name: JobGetStuck :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE state = 'running' + AND id > ? + AND attempted_at < ? +ORDER BY id +LIMIT ? +` + +type JobGetStuckParams struct { + AfterID int64 + StuckHorizon sql.NullTime + Limit int32 +} + +func (q *Queries) JobGetStuck(ctx context.Context, db DBTX, arg *JobGetStuckParams) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobGetStuck, arg.AfterID, arg.StuckHorizon, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobInsertFast = `-- name: JobInsertFast :execresult +INSERT INTO /* TEMPLATE: schema */river_job( + id, + args, + created_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + ?, + CAST(? AS JSON), + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + CAST(? AS JSON), + ?, + ? +) /* TEMPLATE_BEGIN: on_duplicate_key */ ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) /* TEMPLATE_END */ +` + +type JobInsertFastParams struct { + ID sql.NullInt64 + Args dbtype.JSON + CreatedAt interface{} + Kind string + MaxAttempts int64 + Metadata dbtype.JSON + Priority int16 + Queue string + ScheduledAt interface{} + State string + Tags dbtype.JSON + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +func (q *Queries) JobInsertFast(ctx context.Context, db DBTX, arg *JobInsertFastParams) (sql.Result, error) { + return db.ExecContext(ctx, jobInsertFast, + arg.ID, + arg.Args, + arg.CreatedAt, + arg.Kind, + arg.MaxAttempts, + arg.Metadata, + arg.Priority, + arg.Queue, + arg.ScheduledAt, + arg.State, + arg.Tags, + arg.UniqueKey, + arg.UniqueStates, + ) +} + +const jobInsertFullExec = `-- name: JobInsertFullExec :execlastid +INSERT INTO /* TEMPLATE: schema */river_job( + args, + attempt, + attempted_at, + attempted_by, + created_at, + errors, + finalized_at, + kind, + max_attempts, + metadata, + priority, + queue, + scheduled_at, + state, + tags, + unique_key, + unique_states +) VALUES ( + ?, + ?, + ?, + CAST(? AS JSON), + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(? AS JSON), + ?, + ?, + ?, + CAST(? AS JSON), + ?, + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + ?, + CAST(? AS JSON), + ?, + ? +) +` + +type JobInsertFullExecParams struct { + Args dbtype.JSON + Attempt int64 + AttemptedAt sql.NullTime + AttemptedBy dbtype.JSON + CreatedAt interface{} + Errors dbtype.JSON + FinalizedAt sql.NullTime + Kind string + MaxAttempts int64 + Metadata dbtype.JSON + Priority int16 + Queue string + ScheduledAt interface{} + State string + Tags dbtype.JSON + UniqueKey sql.NullString + UniqueStates sql.NullInt16 +} + +func (q *Queries) JobInsertFullExec(ctx context.Context, db DBTX, arg *JobInsertFullExecParams) (int64, error) { + result, err := db.ExecContext(ctx, jobInsertFullExec, + arg.Args, + arg.Attempt, + arg.AttemptedAt, + arg.AttemptedBy, + arg.CreatedAt, + arg.Errors, + arg.FinalizedAt, + arg.Kind, + arg.MaxAttempts, + arg.Metadata, + arg.Priority, + arg.Queue, + arg.ScheduledAt, + arg.State, + arg.Tags, + arg.UniqueKey, + arg.UniqueStates, + ) + if err != nil { + return 0, err + } + return result.LastInsertId() +} + +const jobKindList = `-- name: JobKindList :many +SELECT DISTINCT kind +FROM /* TEMPLATE: schema */river_job +WHERE (? = '' OR LOWER(kind) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(?), '%')) + AND (? = '' OR kind > ?) + AND kind NOT IN (/*SLICE:exclude*/?) +ORDER BY kind ASC +LIMIT ? +` + +type JobKindListParams struct { + Match string + After string + Exclude []string + Limit int32 +} + +func (q *Queries) JobKindList(ctx context.Context, db DBTX, arg *JobKindListParams) ([]string, error) { + query := jobKindList + var queryParams []interface{} + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.After) + queryParams = append(queryParams, arg.After) + if len(arg.Exclude) > 0 { + for _, v := range arg.Exclude { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:exclude*/?", strings.Repeat(",?", len(arg.Exclude))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:exclude*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var kind string + if err := rows.Scan(&kind); err != nil { + return nil, err + } + items = append(items, kind) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobList = `-- name: JobList :many +SELECT id, args, attempt, attempted_at, attempted_by, created_at, errors, finalized_at, kind, max_attempts, metadata, priority, queue, state, scheduled_at, tags, unique_key, unique_states +FROM /* TEMPLATE: schema */river_job +WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ +ORDER BY /* TEMPLATE_BEGIN: order_by_clause */ id /* TEMPLATE_END */ +LIMIT ? +` + +func (q *Queries) JobList(ctx context.Context, db DBTX, limit int32) ([]*RiverJob, error) { + rows, err := db.QueryContext(ctx, jobList, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverJob + for rows.Next() { + var i RiverJob + if err := rows.Scan( + &i.ID, + &i.Args, + &i.Attempt, + &i.AttemptedAt, + &i.AttemptedBy, + &i.CreatedAt, + &i.Errors, + &i.FinalizedAt, + &i.Kind, + &i.MaxAttempts, + &i.Metadata, + &i.Priority, + &i.Queue, + &i.State, + &i.ScheduledAt, + &i.Tags, + &i.UniqueKey, + &i.UniqueStates, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobRescue = `-- name: JobRescue :exec +UPDATE /* TEMPLATE: schema */river_job +SET + errors = JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(? AS JSON)), + finalized_at = ?, + scheduled_at = ?, + metadata = JSON_SET( + metadata, + '$."river:rescue_count"', + COALESCE( + CASE JSON_TYPE(JSON_EXTRACT(metadata, '$."river:rescue_count"')) + WHEN 'INTEGER' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + WHEN 'DOUBLE' THEN JSON_EXTRACT(metadata, '$."river:rescue_count"') + ELSE NULL + END, + 0 + ) + 1 + ), + state = ? +WHERE id = ? + AND state = 'running' + AND attempted_at < ? +` + +type JobRescueParams struct { + Error dbtype.JSON + FinalizedAt sql.NullTime + ScheduledAt time.Time + State string + ID int64 + StuckHorizon sql.NullTime +} + +func (q *Queries) JobRescue(ctx context.Context, db DBTX, arg *JobRescueParams) error { + _, err := db.ExecContext(ctx, jobRescue, + arg.Error, + arg.FinalizedAt, + arg.ScheduledAt, + arg.State, + arg.ID, + arg.StuckHorizon, + ) + return err +} + +const jobRetryExec = `-- name: JobRetryExec :execresult +UPDATE /* TEMPLATE: schema */river_job +SET + state = 'available', + max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END, + finalized_at = NULL, + metadata = JSON_REMOVE(metadata, '$.cancel_attempted_at'), + scheduled_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +WHERE id = ? + AND state != 'running' + AND ( + state <> 'available' + OR scheduled_at > COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ) +` + +type JobRetryExecParams struct { + Now sql.NullTime + ID int64 +} + +func (q *Queries) JobRetryExec(ctx context.Context, db DBTX, arg *JobRetryExecParams) (sql.Result, error) { + return db.ExecContext(ctx, jobRetryExec, arg.Now, arg.ID, arg.Now) +} + +const jobSchedule = `-- name: JobSchedule :many +WITH eligible AS ( + SELECT river_job.id, river_job.unique_key, river_job.unique_states, river_job.priority, river_job.scheduled_at, + CASE + WHEN river_job.unique_key IS NOT NULL AND river_job.unique_states IS NOT NULL THEN + ROW_NUMBER() OVER (PARTITION BY river_job.unique_key ORDER BY river_job.priority, river_job.scheduled_at, river_job.id) + ELSE NULL + END AS row_num + FROM /* TEMPLATE: schema */river_job + WHERE + river_job.state IN ('retryable', 'scheduled') + AND river_job.scheduled_at <= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + ORDER BY + river_job.priority, + river_job.scheduled_at, + river_job.id + LIMIT ? + FOR UPDATE SKIP LOCKED +), +unique_conflicts AS ( + SELECT DISTINCT eligible.unique_key + FROM /* TEMPLATE: schema */river_job + JOIN eligible + ON river_job.unique_key = eligible.unique_key + AND river_job.id != eligible.id + WHERE + river_job.unique_key IS NOT NULL + AND river_job.unique_states IS NOT NULL + AND CASE river_job.state + WHEN 'available' THEN river_job.unique_states & (1 << 0) + WHEN 'cancelled' THEN river_job.unique_states & (1 << 1) + WHEN 'completed' THEN river_job.unique_states & (1 << 2) + WHEN 'discarded' THEN river_job.unique_states & (1 << 3) + WHEN 'pending' THEN river_job.unique_states & (1 << 4) + WHEN 'retryable' THEN river_job.unique_states & (1 << 5) + WHEN 'running' THEN river_job.unique_states & (1 << 6) + WHEN 'scheduled' THEN river_job.unique_states & (1 << 7) + ELSE 0 + END >= 1 +) +SELECT eligible.id, + CASE + WHEN eligible.unique_key IS NULL OR eligible.unique_states IS NULL THEN FALSE + WHEN uc.unique_key IS NOT NULL THEN TRUE + WHEN eligible.row_num > 1 THEN TRUE + ELSE FALSE + END AS conflict_discarded +FROM eligible +LEFT JOIN unique_conflicts uc ON eligible.unique_key = uc.unique_key +ORDER BY eligible.priority, eligible.scheduled_at, eligible.id +` + +type JobScheduleParams struct { + Now sql.NullTime + Limit int32 +} + +type JobScheduleRow struct { + ID int64 + ConflictDiscarded int64 +} + +func (q *Queries) JobSchedule(ctx context.Context, db DBTX, arg *JobScheduleParams) ([]*JobScheduleRow, error) { + rows, err := db.QueryContext(ctx, jobSchedule, arg.Now, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*JobScheduleRow + for rows.Next() { + var i JobScheduleRow + if err := rows.Scan(&i.ID, &i.ConflictDiscarded); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const jobScheduleSetAvailableExec = `-- name: JobScheduleSetAvailableExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET state = 'available' +WHERE id IN (/*SLICE:id*/?) +` + +func (q *Queries) JobScheduleSetAvailableExec(ctx context.Context, db DBTX, id []int64) error { + query := jobScheduleSetAvailableExec + var queryParams []interface{} + if len(id) > 0 { + for _, v := range id { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(id))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobScheduleSetDiscardedExec = `-- name: JobScheduleSetDiscardedExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET metadata = JSON_MERGE_PATCH(metadata, '{"unique_key_conflict": "scheduler_discarded"}'), + finalized_at = COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + state = 'discarded' +WHERE id IN (/*SLICE:id*/?) +` + +type JobScheduleSetDiscardedExecParams struct { + Now sql.NullTime + ID []int64 +} + +func (q *Queries) JobScheduleSetDiscardedExec(ctx context.Context, db DBTX, arg *JobScheduleSetDiscardedExecParams) error { + query := jobScheduleSetDiscardedExec + var queryParams []interface{} + queryParams = append(queryParams, arg.Now) + if len(arg.ID) > 0 { + for _, v := range arg.ID { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:id*/?", strings.Repeat(",?", len(arg.ID))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:id*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const jobSetStateIfRunningExec = `-- name: JobSetStateIfRunningExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE + WHEN state = 'running' + AND ((CAST(? AS CHAR) <> 'available' AND ? <> 'retryable' AND ? <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(? AS SIGNED) + THEN ? ELSE attempt END, + errors = CASE + WHEN state = 'running' AND CAST(? AS SIGNED) + THEN JSON_ARRAY_APPEND(COALESCE(errors, JSON_ARRAY()), '$', CAST(? AS JSON)) + ELSE errors END, + finalized_at = CASE + WHEN state = 'running' AND ((? = 'available' OR ? = 'retryable' OR ? = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + WHEN state = 'running' AND CAST(? AS SIGNED) + THEN ? ELSE finalized_at END, + scheduled_at = CASE + WHEN state = 'running' + AND ((? <> 'available' AND ? <> 'retryable' AND ? <> 'scheduled') OR JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') = 0) + AND CAST(? AS SIGNED) + THEN ? ELSE scheduled_at END, + state = CASE + WHEN state = 'running' AND ((? = 'available' OR ? = 'retryable' OR ? = 'scheduled') AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')) + THEN 'cancelled' + WHEN state = 'running' THEN ? ELSE state END, + metadata = CASE WHEN CAST(? AS SIGNED) + THEN CAST(? AS JSON) ELSE metadata END +WHERE id = ? +` + +type JobSetStateIfRunningExecParams struct { + State string + AttemptDoUpdate int64 + Attempt int64 + ErrorsDoUpdate int64 + Error dbtype.JSON + Now sql.NullTime + FinalizedAtDoUpdate int64 + FinalizedAt sql.NullTime + ScheduledAtDoUpdate int64 + ScheduledAt time.Time + MetadataDoMerge int64 + MetadataUpdates dbtype.JSON + ID int64 +} + +// MySQL evaluates assignments left-to-right. Keep state after the fields whose +// updates depend on it, and metadata last so cancellation checks use the old metadata. +func (q *Queries) JobSetStateIfRunningExec(ctx context.Context, db DBTX, arg *JobSetStateIfRunningExecParams) error { + _, err := db.ExecContext(ctx, jobSetStateIfRunningExec, + arg.State, + arg.State, + arg.State, + arg.AttemptDoUpdate, + arg.Attempt, + arg.ErrorsDoUpdate, + arg.Error, + arg.State, + arg.State, + arg.State, + arg.Now, + arg.FinalizedAtDoUpdate, + arg.FinalizedAt, + arg.State, + arg.State, + arg.State, + arg.ScheduledAtDoUpdate, + arg.ScheduledAt, + arg.State, + arg.State, + arg.State, + arg.State, + arg.MetadataDoMerge, + arg.MetadataUpdates, + arg.ID, + ) + return err +} + +const jobUpdateExec = `-- name: JobUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END +WHERE id = ? +` + +type JobUpdateExecParams struct { + MetadataDoMerge int64 + Metadata dbtype.JSON + ID int64 +} + +func (q *Queries) JobUpdateExec(ctx context.Context, db DBTX, arg *JobUpdateExecParams) error { + _, err := db.ExecContext(ctx, jobUpdateExec, arg.MetadataDoMerge, arg.Metadata, arg.ID) + return err +} + +const jobUpdateFullExec = `-- name: JobUpdateFullExec :exec +UPDATE /* TEMPLATE: schema */river_job +SET + attempt = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE attempt END, + attempted_at = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE attempted_at END, + attempted_by = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE attempted_by END, + errors = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE errors END, + finalized_at = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE finalized_at END, + max_attempts = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE max_attempts END, + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END, + state = CASE WHEN CAST(? AS SIGNED) THEN ? ELSE state END +WHERE id = ? +` + +type JobUpdateFullExecParams struct { + AttemptDoUpdate int64 + Attempt int64 + AttemptedAtDoUpdate int64 + AttemptedAt sql.NullTime + AttemptedByDoUpdate int64 + AttemptedBy dbtype.JSON + ErrorsDoUpdate int64 + Errors dbtype.JSON + FinalizedAtDoUpdate int64 + FinalizedAt sql.NullTime + MaxAttemptsDoUpdate int64 + MaxAttempts int64 + MetadataDoUpdate int64 + Metadata dbtype.JSON + StateDoUpdate int64 + State string + ID int64 +} + +func (q *Queries) JobUpdateFullExec(ctx context.Context, db DBTX, arg *JobUpdateFullExecParams) error { + _, err := db.ExecContext(ctx, jobUpdateFullExec, + arg.AttemptDoUpdate, + arg.Attempt, + arg.AttemptedAtDoUpdate, + arg.AttemptedAt, + arg.AttemptedByDoUpdate, + arg.AttemptedBy, + arg.ErrorsDoUpdate, + arg.Errors, + arg.FinalizedAtDoUpdate, + arg.FinalizedAt, + arg.MaxAttemptsDoUpdate, + arg.MaxAttempts, + arg.MetadataDoUpdate, + arg.Metadata, + arg.StateDoUpdate, + arg.State, + arg.ID, + ) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql new file mode 100644 index 000000000..cee426abd --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql @@ -0,0 +1,60 @@ +CREATE TABLE river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default' PRIMARY KEY +); + +-- name: LeaderAttemptElectExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + leader_id, + elected_at, + expires_at +) VALUES ( + sqlc.arg('leader_id'), + COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +); + +-- name: LeaderAttemptReelectExec :exec +UPDATE /* TEMPLATE: schema */river_leader +SET expires_at = TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +WHERE + elected_at = sqlc.arg('elected_at') + AND expires_at >= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = sqlc.arg('leader_id'); + +-- name: LeaderDeleteExpired :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE expires_at < COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')); + +-- name: LeaderGetElectedLeader :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader; + +-- name: LeaderGetReelected :one +-- Matched renewals can leave expires_at unchanged, which MySQL reports as zero +-- affected rows. Check the current term instead of using that row count. +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +WHERE elected_at = sqlc.arg('elected_at') + AND expires_at >= COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = sqlc.arg('leader_id') +FOR UPDATE; + +-- name: LeaderInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + elected_at, + expires_at, + leader_id +) VALUES ( + COALESCE(sqlc.narg('elected_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + COALESCE(sqlc.narg('expires_at'), TIMESTAMPADD(MICROSECOND, sqlc.arg('ttl'), COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')))), + sqlc.arg('leader_id') +); + +-- name: LeaderResign :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE + elected_at = sqlc.arg('elected_at') + AND leader_id = sqlc.arg('leader_id'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go new file mode 100644 index 000000000..c620a9f4a --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_leader.sql.go @@ -0,0 +1,178 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_leader.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "time" +) + +const leaderAttemptElectExec = `-- name: LeaderAttemptElectExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + leader_id, + elected_at, + expires_at +) VALUES ( + ?, + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +) +` + +type LeaderAttemptElectExecParams struct { + LeaderID string + Now interface{} + TTL int64 +} + +func (q *Queries) LeaderAttemptElectExec(ctx context.Context, db DBTX, arg *LeaderAttemptElectExecParams) error { + _, err := db.ExecContext(ctx, leaderAttemptElectExec, + arg.LeaderID, + arg.Now, + arg.TTL, + arg.Now, + ) + return err +} + +const leaderAttemptReelectExec = `-- name: LeaderAttemptReelectExec :exec +UPDATE /* TEMPLATE: schema */river_leader +SET expires_at = TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00'))) +WHERE + elected_at = ? + AND expires_at >= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = ? +` + +type LeaderAttemptReelectExecParams struct { + TTL int64 + Now sql.NullTime + ElectedAt time.Time + LeaderID string +} + +func (q *Queries) LeaderAttemptReelectExec(ctx context.Context, db DBTX, arg *LeaderAttemptReelectExecParams) error { + _, err := db.ExecContext(ctx, leaderAttemptReelectExec, + arg.TTL, + arg.Now, + arg.ElectedAt, + arg.Now, + arg.LeaderID, + ) + return err +} + +const leaderDeleteExpired = `-- name: LeaderDeleteExpired :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE expires_at < COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +` + +func (q *Queries) LeaderDeleteExpired(ctx context.Context, db DBTX, now sql.NullTime) (int64, error) { + result, err := db.ExecContext(ctx, leaderDeleteExpired, now) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const leaderGetElectedLeader = `-- name: LeaderGetElectedLeader :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +` + +func (q *Queries) LeaderGetElectedLeader(ctx context.Context, db DBTX) (*RiverLeader, error) { + row := db.QueryRowContext(ctx, leaderGetElectedLeader) + var i RiverLeader + err := row.Scan( + &i.ElectedAt, + &i.ExpiresAt, + &i.LeaderID, + &i.Name, + ) + return &i, err +} + +const leaderGetReelected = `-- name: LeaderGetReelected :one +SELECT elected_at, expires_at, leader_id, name +FROM /* TEMPLATE: schema */river_leader +WHERE elected_at = ? + AND expires_at >= COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) + AND leader_id = ? +FOR UPDATE +` + +type LeaderGetReelectedParams struct { + ElectedAt time.Time + Now sql.NullTime + LeaderID string +} + +// Matched renewals can leave expires_at unchanged, which MySQL reports as zero +// affected rows. Check the current term instead of using that row count. +func (q *Queries) LeaderGetReelected(ctx context.Context, db DBTX, arg *LeaderGetReelectedParams) (*RiverLeader, error) { + row := db.QueryRowContext(ctx, leaderGetReelected, arg.ElectedAt, arg.Now, arg.LeaderID) + var i RiverLeader + err := row.Scan( + &i.ElectedAt, + &i.ExpiresAt, + &i.LeaderID, + &i.Name, + ) + return &i, err +} + +const leaderInsertExec = `-- name: LeaderInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_leader ( + elected_at, + expires_at, + leader_id +) VALUES ( + COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + COALESCE(?, TIMESTAMPADD(MICROSECOND, ?, COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')))), + ? +) +` + +type LeaderInsertExecParams struct { + ElectedAt interface{} + Now interface{} + ExpiresAt interface{} + TTL int64 + LeaderID string +} + +func (q *Queries) LeaderInsertExec(ctx context.Context, db DBTX, arg *LeaderInsertExecParams) error { + _, err := db.ExecContext(ctx, leaderInsertExec, + arg.ElectedAt, + arg.Now, + arg.ExpiresAt, + arg.TTL, + arg.Now, + arg.LeaderID, + ) + return err +} + +const leaderResign = `-- name: LeaderResign :execrows +DELETE FROM /* TEMPLATE: schema */river_leader +WHERE + elected_at = ? + AND leader_id = ? +` + +type LeaderResignParams struct { + ElectedAt time.Time + LeaderID string +} + +func (q *Queries) LeaderResign(ctx context.Context, db DBTX, arg *LeaderResignParams) (int64, error) { + result, err := db.ExecContext(ctx, leaderResign, arg.ElectedAt, arg.LeaderID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql new file mode 100644 index 000000000..2c4fffb87 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql @@ -0,0 +1,80 @@ +CREATE TABLE river_migration ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + PRIMARY KEY (line, version) +); + +-- name: RiverMigrationDeleteAssumingMainMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteAssumingMainManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') + AND version IN (sqlc.slice('version')); + +-- name: RiverMigrationDeleteByLineAndVersionManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') + AND version IN (sqlc.slice('version')); + +-- name: RiverMigrationGetAllAssumingMain :many +SELECT + created_at, + version +FROM /* TEMPLATE: schema */river_migration +ORDER BY version; + +-- name: RiverMigrationGetByLine :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') +ORDER BY version; + +-- name: RiverMigrationInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + line, + version +) VALUES ( + sqlc.arg('line'), + sqlc.arg('version') +); + +-- name: RiverMigrationGetByLineAndVersion :one +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') AND version = sqlc.arg('version'); + +-- name: RiverMigrationGetByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = sqlc.arg('line') AND version IN (sqlc.slice('version')) +ORDER BY version; + +-- name: RiverMigrationInsertAssumingMainExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + version +) VALUES ( + sqlc.arg('version') +); + +-- name: RiverMigrationGetByVersion :one +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version = sqlc.arg('version'); + +-- name: RiverMigrationGetByVersionMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (sqlc.slice('version')) +ORDER BY version; diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go new file mode 100644 index 000000000..5238d279f --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_migration.sql.go @@ -0,0 +1,375 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_migration.sql + +package dbsqlc + +import ( + "context" + "strings" + "time" +) + +const riverMigrationDeleteAssumingMainMany = `-- name: RiverMigrationDeleteAssumingMainMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteAssumingMainManyRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationDeleteAssumingMainMany(ctx context.Context, db DBTX, version []int64) ([]*RiverMigrationDeleteAssumingMainManyRow, error) { + query := riverMigrationDeleteAssumingMainMany + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationDeleteAssumingMainManyRow + for rows.Next() { + var i RiverMigrationDeleteAssumingMainManyRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationDeleteAssumingMainManyExec = `-- name: RiverMigrationDeleteAssumingMainManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +` + +func (q *Queries) RiverMigrationDeleteAssumingMainManyExec(ctx context.Context, db DBTX, version []int64) error { + query := riverMigrationDeleteAssumingMainManyExec + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const riverMigrationDeleteByLineAndVersionMany = `-- name: RiverMigrationDeleteByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? + AND version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteByLineAndVersionManyParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationDeleteByLineAndVersionMany(ctx context.Context, db DBTX, arg *RiverMigrationDeleteByLineAndVersionManyParams) ([]*RiverMigration, error) { + query := riverMigrationDeleteByLineAndVersionMany + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationDeleteByLineAndVersionManyExec = `-- name: RiverMigrationDeleteByLineAndVersionManyExec :exec +DELETE FROM /* TEMPLATE: schema */river_migration +WHERE line = ? + AND version IN (/*SLICE:version*/?) +` + +type RiverMigrationDeleteByLineAndVersionManyExecParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationDeleteByLineAndVersionManyExec(ctx context.Context, db DBTX, arg *RiverMigrationDeleteByLineAndVersionManyExecParams) error { + query := riverMigrationDeleteByLineAndVersionManyExec + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const riverMigrationGetAllAssumingMain = `-- name: RiverMigrationGetAllAssumingMain :many +SELECT + created_at, + version +FROM /* TEMPLATE: schema */river_migration +ORDER BY version +` + +type RiverMigrationGetAllAssumingMainRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetAllAssumingMain(ctx context.Context, db DBTX) ([]*RiverMigrationGetAllAssumingMainRow, error) { + rows, err := db.QueryContext(ctx, riverMigrationGetAllAssumingMain) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationGetAllAssumingMainRow + for rows.Next() { + var i RiverMigrationGetAllAssumingMainRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByLine = `-- name: RiverMigrationGetByLine :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? +ORDER BY version +` + +func (q *Queries) RiverMigrationGetByLine(ctx context.Context, db DBTX, line string) ([]*RiverMigration, error) { + rows, err := db.QueryContext(ctx, riverMigrationGetByLine, line) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByLineAndVersion = `-- name: RiverMigrationGetByLineAndVersion :one +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? AND version = ? +` + +type RiverMigrationGetByLineAndVersionParams struct { + Line string + Version int64 +} + +func (q *Queries) RiverMigrationGetByLineAndVersion(ctx context.Context, db DBTX, arg *RiverMigrationGetByLineAndVersionParams) (*RiverMigration, error) { + row := db.QueryRowContext(ctx, riverMigrationGetByLineAndVersion, arg.Line, arg.Version) + var i RiverMigration + err := row.Scan(&i.Line, &i.Version, &i.CreatedAt) + return &i, err +} + +const riverMigrationGetByLineAndVersionMany = `-- name: RiverMigrationGetByLineAndVersionMany :many +SELECT line, version, created_at +FROM /* TEMPLATE: schema */river_migration +WHERE line = ? AND version IN (/*SLICE:version*/?) +ORDER BY version +` + +type RiverMigrationGetByLineAndVersionManyParams struct { + Line string + Version []int64 +} + +func (q *Queries) RiverMigrationGetByLineAndVersionMany(ctx context.Context, db DBTX, arg *RiverMigrationGetByLineAndVersionManyParams) ([]*RiverMigration, error) { + query := riverMigrationGetByLineAndVersionMany + var queryParams []interface{} + queryParams = append(queryParams, arg.Line) + if len(arg.Version) > 0 { + for _, v := range arg.Version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(arg.Version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigration + for rows.Next() { + var i RiverMigration + if err := rows.Scan(&i.Line, &i.Version, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationGetByVersion = `-- name: RiverMigrationGetByVersion :one +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version = ? +` + +type RiverMigrationGetByVersionRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetByVersion(ctx context.Context, db DBTX, version int64) (*RiverMigrationGetByVersionRow, error) { + row := db.QueryRowContext(ctx, riverMigrationGetByVersion, version) + var i RiverMigrationGetByVersionRow + err := row.Scan(&i.CreatedAt, &i.Version) + return &i, err +} + +const riverMigrationGetByVersionMany = `-- name: RiverMigrationGetByVersionMany :many +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration +WHERE version IN (/*SLICE:version*/?) +ORDER BY version +` + +type RiverMigrationGetByVersionManyRow struct { + CreatedAt time.Time + Version int64 +} + +func (q *Queries) RiverMigrationGetByVersionMany(ctx context.Context, db DBTX, version []int64) ([]*RiverMigrationGetByVersionManyRow, error) { + query := riverMigrationGetByVersionMany + var queryParams []interface{} + if len(version) > 0 { + for _, v := range version { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:version*/?", strings.Repeat(",?", len(version))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:version*/?", "NULL", 1) + } + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverMigrationGetByVersionManyRow + for rows.Next() { + var i RiverMigrationGetByVersionManyRow + if err := rows.Scan(&i.CreatedAt, &i.Version); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const riverMigrationInsertAssumingMainExec = `-- name: RiverMigrationInsertAssumingMainExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + version +) VALUES ( + ? +) +` + +func (q *Queries) RiverMigrationInsertAssumingMainExec(ctx context.Context, db DBTX, version int64) error { + _, err := db.ExecContext(ctx, riverMigrationInsertAssumingMainExec, version) + return err +} + +const riverMigrationInsertExec = `-- name: RiverMigrationInsertExec :exec +INSERT INTO /* TEMPLATE: schema */river_migration ( + line, + version +) VALUES ( + ?, + ? +) +` + +type RiverMigrationInsertExecParams struct { + Line string + Version int64 +} + +func (q *Queries) RiverMigrationInsertExec(ctx context.Context, db DBTX, arg *RiverMigrationInsertExecParams) error { + _, err := db.ExecContext(ctx, riverMigrationInsertExec, arg.Line, arg.Version) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql new file mode 100644 index 000000000..4b3de522b --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql @@ -0,0 +1,52 @@ +CREATE TABLE river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +); + +-- name: NotificationDeleteBefore :execrows +DELETE FROM /* TEMPLATE: schema */river_notification +WHERE created_at < sqlc.arg('created_at_horizon') +ORDER BY created_at, id +LIMIT ?; + +-- name: NotificationGetAfter :many +-- InnoDB allocates auto-increment IDs before commit, so IDs may commit out of +-- order. The ascending locking read waits for an earlier uncommitted row rather +-- than returning a later committed row and causing the listener to skip it. +SELECT * +FROM /* TEMPLATE: schema */river_notification +WHERE id > sqlc.arg('after') +ORDER BY id ASC +LIMIT ? +FOR SHARE; + +-- name: NotificationGetIDsForUpdate :many +-- Used to establish a listener's initial high-water mark. This intentionally +-- scans in ascending order instead of using MAX(id) or a descending LIMIT so +-- that it encounters and waits for any lower uncommitted auto-increment IDs. +SELECT id +FROM /* TEMPLATE: schema */river_notification +ORDER BY id +FOR UPDATE; + +-- name: NotificationInsert :exec +INSERT INTO /* TEMPLATE: schema */river_notification ( + payload, + topic +) VALUES ( + sqlc.arg('payload'), + sqlc.arg('topic') +); + +-- name: NotificationInsertJobCancel :exec +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +VALUES ( + JSON_OBJECT('action', 'cancel', 'job_id', CAST(sqlc.arg('job_id') AS SIGNED), 'queue', CAST(sqlc.arg('queue') AS CHAR)), + sqlc.arg('topic') +); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go new file mode 100644 index 000000000..b178e6585 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_notification.sql.go @@ -0,0 +1,148 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_notification.sql + +package dbsqlc + +import ( + "context" + "time" +) + +const notificationDeleteBefore = `-- name: NotificationDeleteBefore :execrows +DELETE FROM /* TEMPLATE: schema */river_notification +WHERE created_at < ? +ORDER BY created_at, id +LIMIT ? +` + +type NotificationDeleteBeforeParams struct { + CreatedAtHorizon time.Time + Limit int32 +} + +func (q *Queries) NotificationDeleteBefore(ctx context.Context, db DBTX, arg *NotificationDeleteBeforeParams) (int64, error) { + result, err := db.ExecContext(ctx, notificationDeleteBefore, arg.CreatedAtHorizon, arg.Limit) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const notificationGetAfter = `-- name: NotificationGetAfter :many +SELECT id, created_at, payload, topic +FROM /* TEMPLATE: schema */river_notification +WHERE id > ? +ORDER BY id ASC +LIMIT ? +FOR SHARE +` + +type NotificationGetAfterParams struct { + After int64 + Limit int32 +} + +// InnoDB allocates auto-increment IDs before commit, so IDs may commit out of +// order. The ascending locking read waits for an earlier uncommitted row rather +// than returning a later committed row and causing the listener to skip it. +func (q *Queries) NotificationGetAfter(ctx context.Context, db DBTX, arg *NotificationGetAfterParams) ([]*RiverNotification, error) { + rows, err := db.QueryContext(ctx, notificationGetAfter, arg.After, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverNotification + for rows.Next() { + var i RiverNotification + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.Payload, + &i.Topic, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const notificationGetIDsForUpdate = `-- name: NotificationGetIDsForUpdate :many +SELECT id +FROM /* TEMPLATE: schema */river_notification +ORDER BY id +FOR UPDATE +` + +// Used to establish a listener's initial high-water mark. This intentionally +// scans in ascending order instead of using MAX(id) or a descending LIMIT so +// that it encounters and waits for any lower uncommitted auto-increment IDs. +func (q *Queries) NotificationGetIDsForUpdate(ctx context.Context, db DBTX) ([]int64, error) { + rows, err := db.QueryContext(ctx, notificationGetIDsForUpdate) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const notificationInsert = `-- name: NotificationInsert :exec +INSERT INTO /* TEMPLATE: schema */river_notification ( + payload, + topic +) VALUES ( + ?, + ? +) +` + +type NotificationInsertParams struct { + Payload string + Topic string +} + +func (q *Queries) NotificationInsert(ctx context.Context, db DBTX, arg *NotificationInsertParams) error { + _, err := db.ExecContext(ctx, notificationInsert, arg.Payload, arg.Topic) + return err +} + +const notificationInsertJobCancel = `-- name: NotificationInsertJobCancel :exec +INSERT INTO /* TEMPLATE: schema */river_notification (payload, topic) +VALUES ( + JSON_OBJECT('action', 'cancel', 'job_id', CAST(? AS SIGNED), 'queue', CAST(? AS CHAR)), + ? +) +` + +type NotificationInsertJobCancelParams struct { + JobID int64 + Queue interface{} + Topic string +} + +func (q *Queries) NotificationInsertJobCancel(ctx context.Context, db DBTX, arg *NotificationInsertJobCancelParams) error { + _, err := db.ExecContext(ctx, notificationInsertJobCancel, arg.JobID, arg.Queue, arg.Topic) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql new file mode 100644 index 000000000..c7202e76c --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql @@ -0,0 +1,101 @@ +CREATE TABLE river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + -- sqlc's MySQL parser doesn't accept UTC_TIMESTAMP(6) in a DEFAULT + -- expression. Runtime migrations use UTC_TIMESTAMP(6); this codegen-only + -- schema declaration uses NOW(6). + created_at DATETIME(6) NOT NULL DEFAULT (NOW(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL DEFAULT (NOW(6)) +); + +-- name: QueueCreateOrSetUpdatedAtExec :exec +INSERT INTO /* TEMPLATE: schema */river_queue ( + created_at, + metadata, + name, + paused_at, + updated_at +) VALUES ( + COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(sqlc.arg('metadata') AS JSON), + sqlc.arg('name'), + sqlc.narg('paused_at'), + COALESCE(sqlc.narg('updated_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +) ON DUPLICATE KEY UPDATE + updated_at = COALESCE(sqlc.narg('updated_at'), sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')); + +-- name: QueueGet :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = sqlc.arg('name'); + +-- name: QueueGetForUpdate :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = sqlc.arg('name') +FOR UPDATE; + +-- name: QueueDeleteExpiredSelect :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE updated_at < sqlc.arg('updated_at_horizon') +ORDER BY name ASC +LIMIT ? +FOR UPDATE; + +-- name: QueueDeleteExpiredExec :exec +DELETE FROM /* TEMPLATE: schema */river_queue +WHERE name IN (sqlc.slice('names')); + +-- name: QueueList :many +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +ORDER BY name ASC +LIMIT ?; + +-- name: QueueNameList :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE + name > sqlc.arg('after') + AND (sqlc.arg('match') = '' OR LOWER(name) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(sqlc.arg('match')), '%')) + AND name NOT IN (sqlc.slice('exclude')) +ORDER BY name ASC +LIMIT ?; + +-- MySQL evaluates SET clauses left-to-right using already-updated values, so +-- updated_at must be set BEFORE paused_at to see the original paused_at value. + +-- name: QueuePauseAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END; + +-- name: QueuePauseByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +WHERE name = sqlc.arg('name'); + +-- name: QueueResumeAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL; + +-- name: QueueResumeByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(sqlc.narg('now'), CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +WHERE name = sqlc.arg('name'); + +-- name: QueueUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_queue +SET + metadata = CASE WHEN CAST(sqlc.arg('metadata_do_update') AS SIGNED) THEN CAST(sqlc.arg('metadata') AS JSON) ELSE metadata END, + updated_at = CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00') +WHERE name = sqlc.arg('name'); diff --git a/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go new file mode 100644 index 000000000..f209f998e --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/river_queue.sql.go @@ -0,0 +1,323 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: river_queue.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" + "time" + + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbtype" +) + +const queueCreateOrSetUpdatedAtExec = `-- name: QueueCreateOrSetUpdatedAtExec :exec +INSERT INTO /* TEMPLATE: schema */river_queue ( + created_at, + metadata, + name, + paused_at, + updated_at +) VALUES ( + COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')), + CAST(? AS JSON), + ?, + ?, + COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +) ON DUPLICATE KEY UPDATE + updated_at = COALESCE(?, ?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) +` + +type QueueCreateOrSetUpdatedAtExecParams struct { + Now sql.NullTime + Metadata dbtype.JSON + Name string + PausedAt sql.NullTime + UpdatedAt sql.NullTime +} + +func (q *Queries) QueueCreateOrSetUpdatedAtExec(ctx context.Context, db DBTX, arg *QueueCreateOrSetUpdatedAtExecParams) error { + _, err := db.ExecContext(ctx, queueCreateOrSetUpdatedAtExec, + arg.Now, + arg.Metadata, + arg.Name, + arg.PausedAt, + arg.UpdatedAt, + arg.Now, + arg.UpdatedAt, + arg.Now, + ) + return err +} + +const queueDeleteExpiredExec = `-- name: QueueDeleteExpiredExec :exec +DELETE FROM /* TEMPLATE: schema */river_queue +WHERE name IN (/*SLICE:names*/?) +` + +func (q *Queries) QueueDeleteExpiredExec(ctx context.Context, db DBTX, names []string) error { + query := queueDeleteExpiredExec + var queryParams []interface{} + if len(names) > 0 { + for _, v := range names { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:names*/?", strings.Repeat(",?", len(names))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:names*/?", "NULL", 1) + } + _, err := db.ExecContext(ctx, query, queryParams...) + return err +} + +const queueDeleteExpiredSelect = `-- name: QueueDeleteExpiredSelect :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE updated_at < ? +ORDER BY name ASC +LIMIT ? +FOR UPDATE +` + +type QueueDeleteExpiredSelectParams struct { + UpdatedAtHorizon time.Time + Limit int32 +} + +func (q *Queries) QueueDeleteExpiredSelect(ctx context.Context, db DBTX, arg *QueueDeleteExpiredSelectParams) ([]string, error) { + rows, err := db.QueryContext(ctx, queueDeleteExpiredSelect, arg.UpdatedAtHorizon, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + items = append(items, name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queueGet = `-- name: QueueGet :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = ? +` + +func (q *Queries) QueueGet(ctx context.Context, db DBTX, name string) (*RiverQueue, error) { + row := db.QueryRowContext(ctx, queueGet, name) + var i RiverQueue + err := row.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ) + return &i, err +} + +const queueGetForUpdate = `-- name: QueueGetForUpdate :one +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +WHERE name = ? +FOR UPDATE +` + +func (q *Queries) QueueGetForUpdate(ctx context.Context, db DBTX, name string) (*RiverQueue, error) { + row := db.QueryRowContext(ctx, queueGetForUpdate, name) + var i RiverQueue + err := row.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ) + return &i, err +} + +const queueList = `-- name: QueueList :many +SELECT name, created_at, metadata, paused_at, updated_at +FROM /* TEMPLATE: schema */river_queue +ORDER BY name ASC +LIMIT ? +` + +func (q *Queries) QueueList(ctx context.Context, db DBTX, limit int32) ([]*RiverQueue, error) { + rows, err := db.QueryContext(ctx, queueList, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*RiverQueue + for rows.Next() { + var i RiverQueue + if err := rows.Scan( + &i.Name, + &i.CreatedAt, + &i.Metadata, + &i.PausedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queueNameList = `-- name: QueueNameList :many +SELECT name +FROM /* TEMPLATE: schema */river_queue +WHERE + name > ? + AND (? = '' OR LOWER(name) COLLATE utf8mb4_general_ci LIKE CONCAT('%', LOWER(?), '%')) + AND name NOT IN (/*SLICE:exclude*/?) +ORDER BY name ASC +LIMIT ? +` + +type QueueNameListParams struct { + After string + Match string + Exclude []string + Limit int32 +} + +func (q *Queries) QueueNameList(ctx context.Context, db DBTX, arg *QueueNameListParams) ([]string, error) { + query := queueNameList + var queryParams []interface{} + queryParams = append(queryParams, arg.After) + queryParams = append(queryParams, arg.Match) + queryParams = append(queryParams, arg.Match) + if len(arg.Exclude) > 0 { + for _, v := range arg.Exclude { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:exclude*/?", strings.Repeat(",?", len(arg.Exclude))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:exclude*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Limit) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + items = append(items, name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const queuePauseAll = `-- name: QueuePauseAll :execresult + +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +` + +type QueuePauseAllParams struct { + Now sql.NullTime +} + +// MySQL evaluates SET clauses left-to-right using already-updated values, so +// updated_at must be set BEFORE paused_at to see the original paused_at value. +func (q *Queries) QueuePauseAll(ctx context.Context, db DBTX, arg *QueuePauseAllParams) (sql.Result, error) { + return db.ExecContext(ctx, queuePauseAll, arg.Now, arg.Now) +} + +const queuePauseByName = `-- name: QueuePauseByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = CASE WHEN paused_at IS NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE paused_at END +WHERE name = ? +` + +type QueuePauseByNameParams struct { + Now sql.NullTime + Name string +} + +func (q *Queries) QueuePauseByName(ctx context.Context, db DBTX, arg *QueuePauseByNameParams) (sql.Result, error) { + return db.ExecContext(ctx, queuePauseByName, arg.Now, arg.Now, arg.Name) +} + +const queueResumeAll = `-- name: QueueResumeAll :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +` + +func (q *Queries) QueueResumeAll(ctx context.Context, db DBTX, now sql.NullTime) (sql.Result, error) { + return db.ExecContext(ctx, queueResumeAll, now) +} + +const queueResumeByName = `-- name: QueueResumeByName :execresult +UPDATE /* TEMPLATE: schema */river_queue +SET + updated_at = CASE WHEN paused_at IS NOT NULL THEN COALESCE(?, CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00')) ELSE updated_at END, + paused_at = NULL +WHERE name = ? +` + +type QueueResumeByNameParams struct { + Now sql.NullTime + Name string +} + +func (q *Queries) QueueResumeByName(ctx context.Context, db DBTX, arg *QueueResumeByNameParams) (sql.Result, error) { + return db.ExecContext(ctx, queueResumeByName, arg.Now, arg.Name) +} + +const queueUpdateExec = `-- name: QueueUpdateExec :exec +UPDATE /* TEMPLATE: schema */river_queue +SET + metadata = CASE WHEN CAST(? AS SIGNED) THEN CAST(? AS JSON) ELSE metadata END, + updated_at = CONVERT_TZ(NOW(6), @@session.time_zone, '+00:00') +WHERE name = ? +` + +type QueueUpdateExecParams struct { + MetadataDoUpdate int64 + Metadata dbtype.JSON + Name string +} + +func (q *Queries) QueueUpdateExec(ctx context.Context, db DBTX, arg *QueueUpdateExecParams) error { + _, err := db.ExecContext(ctx, queueUpdateExec, arg.MetadataDoUpdate, arg.Metadata, arg.Name) + return err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/schema.sql b/riverdriver/rivermysql/internal/dbsqlc/schema.sql new file mode 100644 index 000000000..877f9460c --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/schema.sql @@ -0,0 +1,79 @@ +-- Dummy table definitions for INFORMATION_SCHEMA system tables so sqlc can +-- resolve column types. At runtime the template prefix replaces the empty +-- default with "INFORMATION_SCHEMA.", making queries target the real system +-- tables. +CREATE TABLE COLUMNS ( + COLUMN_NAME VARCHAR(128) NOT NULL, + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +CREATE TABLE SCHEMATA ( + SCHEMA_NAME VARCHAR(128) NOT NULL +); + +CREATE TABLE STATISTICS ( + INDEX_NAME VARCHAR(128) NOT NULL, + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +CREATE TABLE TABLES ( + TABLE_NAME VARCHAR(128) NOT NULL, + TABLE_SCHEMA VARCHAR(128) NOT NULL +); + +-- name: ColumnExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */COLUMNS + WHERE COLUMN_NAME = sqlc.arg('column_name') + AND TABLE_NAME = sqlc.arg('table_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); + +-- name: IndexExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */STATISTICS + WHERE INDEX_NAME = sqlc.arg('index_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); + +-- name: IndexGetTableName :one +SELECT TABLE_NAME +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME = sqlc.arg('index_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +LIMIT 1; + +-- name: IndexReindexArtifacts :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) + AND REGEXP_LIKE(INDEX_NAME, sqlc.arg('artifact_pattern'), 'c') +ORDER BY INDEX_NAME; + +-- name: IndexesExist :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME IN (sqlc.slice('index_names')) + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()); + +-- name: Ping :one +SELECT 1; + +-- name: SchemaGetExpired :many +SELECT SCHEMA_NAME +FROM /* TEMPLATE: information_schema */SCHEMATA +WHERE SCHEMA_NAME LIKE CONCAT(sqlc.arg('prefix'), '%') + AND SCHEMA_NAME < sqlc.arg('before_name') +ORDER BY SCHEMA_NAME; + +-- name: TableExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */TABLES + WHERE TABLE_NAME = sqlc.arg('table_name') + AND TABLE_SCHEMA = COALESCE(sqlc.narg('schema'), DATABASE()) +); diff --git a/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go b/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go new file mode 100644 index 000000000..d1ea7467e --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/schema.sql.go @@ -0,0 +1,226 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: schema.sql + +package dbsqlc + +import ( + "context" + "database/sql" + "strings" +) + +const columnExists = `-- name: ColumnExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */COLUMNS + WHERE COLUMN_NAME = ? + AND TABLE_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type ColumnExistsParams struct { + ColumnName string + TableName string + Schema sql.NullString +} + +func (q *Queries) ColumnExists(ctx context.Context, db DBTX, arg *ColumnExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, columnExists, arg.ColumnName, arg.TableName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const indexExists = `-- name: IndexExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */STATISTICS + WHERE INDEX_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type IndexExistsParams struct { + IndexName string + Schema sql.NullString +} + +func (q *Queries) IndexExists(ctx context.Context, db DBTX, arg *IndexExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, indexExists, arg.IndexName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const indexGetTableName = `-- name: IndexGetTableName :one +SELECT TABLE_NAME +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +LIMIT 1 +` + +type IndexGetTableNameParams struct { + IndexName string + Schema sql.NullString +} + +func (q *Queries) IndexGetTableName(ctx context.Context, db DBTX, arg *IndexGetTableNameParams) (string, error) { + row := db.QueryRowContext(ctx, indexGetTableName, arg.IndexName, arg.Schema) + var table_name string + err := row.Scan(&table_name) + return table_name, err +} + +const indexReindexArtifacts = `-- name: IndexReindexArtifacts :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) + AND REGEXP_LIKE(INDEX_NAME, ?, 'c') +ORDER BY INDEX_NAME +` + +type IndexReindexArtifactsParams struct { + Schema sql.NullString + ArtifactPattern string +} + +func (q *Queries) IndexReindexArtifacts(ctx context.Context, db DBTX, arg *IndexReindexArtifactsParams) ([]string, error) { + rows, err := db.QueryContext(ctx, indexReindexArtifacts, arg.Schema, arg.ArtifactPattern) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var index_name string + if err := rows.Scan(&index_name); err != nil { + return nil, err + } + items = append(items, index_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const indexesExist = `-- name: IndexesExist :many +SELECT DISTINCT INDEX_NAME AS index_name +FROM /* TEMPLATE: information_schema */STATISTICS +WHERE INDEX_NAME IN (/*SLICE:index_names*/?) + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +` + +type IndexesExistParams struct { + IndexNames []string + Schema sql.NullString +} + +func (q *Queries) IndexesExist(ctx context.Context, db DBTX, arg *IndexesExistParams) ([]string, error) { + query := indexesExist + var queryParams []interface{} + if len(arg.IndexNames) > 0 { + for _, v := range arg.IndexNames { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:index_names*/?", strings.Repeat(",?", len(arg.IndexNames))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:index_names*/?", "NULL", 1) + } + queryParams = append(queryParams, arg.Schema) + rows, err := db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var index_name string + if err := rows.Scan(&index_name); err != nil { + return nil, err + } + items = append(items, index_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const ping = `-- name: Ping :one +SELECT 1 +` + +func (q *Queries) Ping(ctx context.Context, db DBTX) (int64, error) { + row := db.QueryRowContext(ctx, ping) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const schemaGetExpired = `-- name: SchemaGetExpired :many +SELECT SCHEMA_NAME +FROM /* TEMPLATE: information_schema */SCHEMATA +WHERE SCHEMA_NAME LIKE CONCAT(?, '%') + AND SCHEMA_NAME < ? +ORDER BY SCHEMA_NAME +` + +type SchemaGetExpiredParams struct { + Prefix interface{} + BeforeName string +} + +func (q *Queries) SchemaGetExpired(ctx context.Context, db DBTX, arg *SchemaGetExpiredParams) ([]string, error) { + rows, err := db.QueryContext(ctx, schemaGetExpired, arg.Prefix, arg.BeforeName) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var schema_name string + if err := rows.Scan(&schema_name); err != nil { + return nil, err + } + items = append(items, schema_name) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const tableExists = `-- name: TableExists :one +SELECT EXISTS ( + SELECT 1 + FROM /* TEMPLATE: information_schema */TABLES + WHERE TABLE_NAME = ? + AND TABLE_SCHEMA = COALESCE(?, DATABASE()) +) +` + +type TableExistsParams struct { + TableName string + Schema sql.NullString +} + +func (q *Queries) TableExists(ctx context.Context, db DBTX, arg *TableExistsParams) (bool, error) { + row := db.QueryRowContext(ctx, tableExists, arg.TableName, arg.Schema) + var exists bool + err := row.Scan(&exists) + return exists, err +} diff --git a/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml b/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml new file mode 100644 index 000000000..af2411eeb --- /dev/null +++ b/riverdriver/rivermysql/internal/dbsqlc/sqlc.yaml @@ -0,0 +1,43 @@ +version: "2" +sql: + - engine: "mysql" + queries: + - river_job.sql + - river_leader.sql + - river_migration.sql + - river_notification.sql + - river_queue.sql + - schema.sql + schema: + - river_job.sql + - river_leader.sql + - river_migration.sql + - river_notification.sql + - river_queue.sql + - schema.sql + gen: + go: + package: "dbsqlc" + out: "." + emit_exact_table_names: true + emit_methods_with_db_argument: true + emit_params_struct_pointers: true + emit_pointers_for_null_types: true + emit_result_struct_pointers: true + + rename: + ids: "IDs" + ttl: "TTL" + + overrides: + - db_type: "json" + go_type: + import: "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbtype" + type: "JSON" + - db_type: "json" + go_type: + import: "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbtype" + type: "JSON" + nullable: true + - db_type: "int" + go_type: "int64" diff --git a/riverdriver/rivermysql/internal/dbtype/json.go b/riverdriver/rivermysql/internal/dbtype/json.go new file mode 100644 index 000000000..81c44abf0 --- /dev/null +++ b/riverdriver/rivermysql/internal/dbtype/json.go @@ -0,0 +1,37 @@ +// Package dbtype provides MySQL-specific SQL parameter types. +package dbtype + +import ( + "bytes" + "database/sql/driver" + "fmt" +) + +// JSON is a JSON document bound as text. go-sql-driver/mysql interpolates a +// []byte parameter as a binary string, which MySQL rejects as JSON. Binding +// text works with both prepared statements and interpolateParams=true, while +// retaining []byte's scanning and decoding behavior. +type JSON []byte //nolint:recvcheck // Scan must mutate a pointer; Value must also work on non-pointer parameters. + +// Scan copies a document out of the driver's reusable result buffer. +func (j *JSON) Scan(value any) error { + switch value := value.(type) { + case nil: + *j = nil + case []byte: + *j = bytes.Clone(value) + case string: + *j = []byte(value) + default: + return fmt.Errorf("cannot scan %T into JSON", value) + } + return nil +} + +// Value preserves SQL NULL and otherwise binds the document as text. +func (j JSON) Value() (driver.Value, error) { + if j == nil { + return nil, nil //nolint:nilnil // A nil driver.Value represents SQL NULL. + } + return string(j), nil +} diff --git a/riverdriver/rivermysql/internal/dbtype/json_test.go b/riverdriver/rivermysql/internal/dbtype/json_test.go new file mode 100644 index 000000000..61db7075b --- /dev/null +++ b/riverdriver/rivermysql/internal/dbtype/json_test.go @@ -0,0 +1,55 @@ +package dbtype + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestJSON(t *testing.T) { + t.Parallel() + + t.Run("ScanCopiesDriverBuffer", func(t *testing.T) { + t.Parallel() + + buffer := []byte(`{"value":1}`) + var value JSON + require.NoError(t, value.Scan(buffer)) + buffer[9] = '2' + require.Equal(t, JSON(`{"value":1}`), value) + }) + + t.Run("ScanRejectsUnexpectedTypes", func(t *testing.T) { + t.Parallel() + + var value JSON + require.ErrorContains(t, value.Scan(42), "cannot scan int into JSON") + }) + + t.Run("ScanSQLNull", func(t *testing.T) { + t.Parallel() + + value := JSON(`null`) + require.NoError(t, value.Scan(nil)) + require.Nil(t, value) + }) + + t.Run("ScanText", func(t *testing.T) { + t.Parallel() + + var value JSON + require.NoError(t, value.Scan(`{"value":1}`)) + require.Equal(t, JSON(`{"value":1}`), value) + }) + + t.Run("ValueDistinguishesSQLNullAndJSONNull", func(t *testing.T) { + t.Parallel() + + value, err := JSON(nil).Value() + require.NoError(t, err) + require.Nil(t, value) + value, err = JSON(`null`).Value() + require.NoError(t, err) + require.Equal(t, "null", value) + }) +} diff --git a/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql b/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/riverdriver/rivermysql/migration/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql b/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..8a76b2634 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql b/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/riverdriver/rivermysql/migration/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql b/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql new file mode 100644 index 000000000..f841bf89a --- /dev/null +++ b/riverdriver/rivermysql/migration/main/002_initial_schema.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL has no LISTEN/NOTIFY transport, so river_job_notify is omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql b/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql b/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..b544e5000 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. States use VARCHAR with an +-- explicit constraint, so changing them does not require a table rebuild. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql b/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..5906e5567 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql b/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..b1e83547d --- /dev/null +++ b/riverdriver/rivermysql/migration/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql b/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/riverdriver/rivermysql/migration/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql b/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/riverdriver/rivermysql/migration/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..1f368209e --- /dev/null +++ b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..82ec71552 --- /dev/null +++ b/riverdriver/rivermysql/migration/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/riverdriver/rivermysql/migration/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/riverdriver/rivermysql/river_mysql_driver.go b/riverdriver/rivermysql/river_mysql_driver.go new file mode 100644 index 000000000..465f3ee93 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver.go @@ -0,0 +1,2240 @@ +// Package rivermysql provides a River driver implementation for MySQL. +// +// This driver targets MySQL 8.0+ and requires `go-sql-driver/mysql` or a +// compatible driver registered with `database/sql`. DSNs used with River +// should include `parseTime=true`, `loc=UTC`, and `multiStatements=true`. +// `parseTime` and `loc` ensure `DATETIME` columns are scanned as UTC +// [time.Time] values, while `multiStatements` is needed to run River's +// migrations. +// +// With go-sql-driver/mysql, the optional `interpolateParams=true` setting +// avoids preparing and closing a server-side statement for every query. +// Benchmark it with your workload, particularly when database round-trip +// latency is significant. River preserves the supplied pool's configuration. +// +// MySQL does not support LISTEN/NOTIFY, so this driver uses a +// `river_notification` outbox table for notifications. It also does not support +// `RETURNING` clauses, so most write operations are carried out as two-step +// operations (write + read). +// +// This driver is currently in early development. It's exercised in the test +// suite, but has minimal real world use as of yet. +package rivermysql + +import ( + "bytes" + "context" + "database/sql" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" + "regexp" + "slices" + "strings" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" + "github.com/riverqueue/river/rivershared/uniquestates" + "github.com/riverqueue/river/rivershared/util/dbutil" + "github.com/riverqueue/river/rivershared/util/randutil" + "github.com/riverqueue/river/rivershared/util/savepointutil" + "github.com/riverqueue/river/rivershared/util/sliceutil" + "github.com/riverqueue/river/rivertype" +) + +//go:embed migration/*/*.sql +var migrationFS embed.FS + +// Driver is an implementation of riverdriver.Driver for MySQL. +type Driver struct { + dbPool *sql.DB + replacer sqlctemplate.Replacer +} + +// New returns a new MySQL driver for use with River. +// +// It takes an sql.DB to use with River. The DSN should include +// `parseTime=true&loc=UTC&multiStatements=true`; see the package documentation +// for details. The pool must not be closed while associated River objects are +// running. +func New(dbPool *sql.DB) *Driver { + return &Driver{ + dbPool: dbPool, + replacer: sqlctemplate.Replacer{UnnumberedPlaceholders: true}, + } +} + +const argPlaceholder = "?" + +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameMySQL } +func (d *Driver) SafeIdentifier(ident string) string { return mysqlIdentifier(ident) } + +func (d *Driver) GetExecutor() riverdriver.Executor { + return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d, nil} +} + +func (d *Driver) GetListener(params *riverdriver.GetListenenerParams) riverdriver.Listener { + return &Listener{ + dbPool: d.dbPool, + pollInterval: notificationPollIntervalDefault, + replacer: &d.replacer, + schema: params.Schema, + topics: make(map[string]int64), + } +} + +func (d *Driver) GetMigrationDefaultLines() []string { return []string{riverdriver.MigrationLineMain} } + +func (d *Driver) GetMigrationFS(line string) fs.FS { + if line == riverdriver.MigrationLineMain { + return migrationFS + } + panic("migration line does not exist: " + line) +} +func (d *Driver) GetMigrationLines() []string { return []string{riverdriver.MigrationLineMain} } +func (d *Driver) GetMigrationTruncateTables(line string, version int) []string { + if line == riverdriver.MigrationLineMain { + return riverdriver.MigrationLineMainTruncateTables(version) + } + panic("migration line does not exist: " + line) +} + +func (d *Driver) PoolIsSet() bool { return d.dbPool != nil } +func (d *Driver) PoolSet(dbPool any) error { + if d.dbPool != nil { + return errors.New("cannot PoolSet when internal pool is already non-nil") + } + d.dbPool = dbPool.(*sql.DB) //nolint:forcetypeassert + return nil +} + +func (d *Driver) SQLFragmentColumnContainsAll(column, namedArg string, values []string) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + return fmt.Sprintf("JSON_CONTAINS(%s, CAST(@%s AS JSON))", column, namedArg), string(arg), nil +} + +func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values []string) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + return fmt.Sprintf("EXISTS (SELECT 1 FROM JSON_TABLE(CAST(@%s AS JSON), '$[*]' COLUMNS(val VARCHAR(255) PATH '$')) AS filter_value WHERE JSON_CONTAINS(%s, JSON_QUOTE(filter_value.val)))", namedArg, column), string(arg), nil +} + +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + if !json.Valid([]byte(fragment)) { + return "", nil, errors.New("invalid JSON metadata fragment") + } + args := make(map[string]any) + bindFunc := func(value any) string { + name := fmt.Sprintf("%s_%d", namedArg, len(args)) + args[name] = value + return "@" + name + } + predicate, err := jsonContains(column, json.RawMessage(fragment), bindFunc, 0) + return predicate, args, err +} + +func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { + arg, err := json.Marshal(values) + if err != nil { + return "", nil, err + } + + // Match integer types to avoid implicit string-to-double conversion losing + // precision for large IDs. Strings use River's case-sensitive collation. + valueType := "VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin" + switch values.(type) { + case []int, []int8, []int16, []int32, []int64: + valueType = "BIGINT" + } + + return fmt.Sprintf("%s IN (SELECT jt.val FROM JSON_TABLE(CAST(@%s AS JSON), '$[*]' COLUMNS(val %s PATH '$')) AS jt)", column, column, valueType), string(arg), nil +} + +func (d *Driver) SupportsListener() bool { return true } +func (d *Driver) SupportsListenNotify() bool { return true } +func (d *Driver) TimePrecision() time.Duration { return time.Microsecond } + +func (d *Driver) UnwrapExecutor(tx *sql.Tx) riverdriver.ExecutorTx { + // Allows UnwrapExecutor to be invoked even if driver is nil. + var replacer *sqlctemplate.Replacer + if d == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } else { + replacer = &d.replacer + } + + executorTx := &ExecutorTx{tx: tx} + executorTx.Executor = Executor{nil, templateReplaceWrapper{tx, replacer}, d, executorTx} + + return executorTx +} + +func (d *Driver) UnwrapTx(execTx riverdriver.ExecutorTx) *sql.Tx { + switch execTx := execTx.(type) { + case *ExecutorSubTx: + return execTx.tx + case *ExecutorTx: + return execTx.tx + } + panic("unhandled executor type") +} + +type Executor struct { + dbPool *sql.DB + dbtx templateReplaceWrapper + driver *Driver + execTx riverdriver.ExecutorTx +} + +func (e *Executor) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + if e.execTx != nil { + return e.execTx.Begin(ctx) + } + + tx, err := e.dbPool.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return nil, err + } + + executorTx := &ExecutorTx{tx: tx} + executorTx.Executor = Executor{nil, templateReplaceWrapper{tx, &e.driver.replacer}, e.driver, executorTx} + return executorTx, nil +} + +func (e *Executor) ColumnExists(ctx context.Context, params *riverdriver.ColumnExistsParams) (bool, error) { + exists, err := dbsqlc.New().ColumnExists(informationSchemaParam(ctx), e.dbtx, &dbsqlc.ColumnExistsParams{ + ColumnName: params.Column, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + TableName: params.Table, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) Exec(ctx context.Context, sql string, args ...any) error { + _, err := e.dbtx.ExecContext(ctx, sql, args...) + return interpretError(err) +} + +func (e *Executor) IndexDropIfExists(ctx context.Context, params *riverdriver.IndexDropIfExistsParams) error { + indexName := strings.TrimSpace(params.Index) + + exists, err := e.IndexExists(ctx, &riverdriver.IndexExistsParams{ + Index: indexName, + Schema: params.Schema, + }) + if err != nil { + return err + } + if !exists { + return nil + } + + // MySQL's DROP INDEX requires the table name. Look it up from the index. + tableName, err := dbsqlc.New().IndexGetTableName(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexGetTableNameParams{ + IndexName: indexName, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return interpretError(err) + } + + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + _, err = e.dbtx.ExecContext(ctx, "DROP INDEX "+mysqlIdentifier(indexName)+" ON "+maybeSchema+mysqlIdentifier(tableName)) + return interpretError(err) +} + +func (e *Executor) IndexExists(ctx context.Context, params *riverdriver.IndexExistsParams) (bool, error) { + ctx = informationSchemaParam(ctx) + + exists, err := dbsqlc.New().IndexExists(ctx, e.dbtx, &dbsqlc.IndexExistsParams{ + IndexName: params.Index, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) IndexReindex(ctx context.Context, params *riverdriver.IndexReindexParams) error { + // MySQL has no per-index REINDEX operation. ANALYZE TABLE is the safe + // online approximation: it refreshes the selected index's table statistics + // without rebuilding and locking the entire table as OPTIMIZE TABLE would. + tableName, err := dbsqlc.New().IndexGetTableName(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexGetTableNameParams{ + IndexName: params.Index, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return interpretError(err) + } + + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + _, err = e.dbtx.ExecContext(ctx, "ANALYZE TABLE "+maybeSchema+mysqlIdentifier(tableName)) + return interpretError(err) +} + +func (e *Executor) IndexReindexArtifacts(ctx context.Context, params *riverdriver.IndexReindexArtifactsParams) ([]string, error) { + artifacts, err := dbsqlc.New().IndexReindexArtifacts(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexReindexArtifactsParams{ + ArtifactPattern: "^" + regexp.QuoteMeta(params.Index) + `_(ccnew|ccold)[0-9]*$`, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + return artifacts, interpretError(err) +} + +func (e *Executor) IndexesExist(ctx context.Context, params *riverdriver.IndexesExistParams) (map[string]bool, error) { + foundNames, err := dbsqlc.New().IndexesExist(informationSchemaParam(ctx), e.dbtx, &dbsqlc.IndexesExistParams{ + IndexNames: params.IndexNames, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return nil, interpretError(err) + } + + exists := make(map[string]bool, len(params.IndexNames)) + for _, name := range foundNames { + exists[name] = true + } + return exists, nil +} + +func (e *Executor) InitDriver(ctx context.Context) error { return nil } + +func (e *Executor) JobCancel(ctx context.Context, params *riverdriver.JobCancelParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + res, err := dbsqlc.New().JobCancelExec(ctx, dbtx, &dbsqlc.JobCancelExecParams{ + ID: params.ID, + CancelAttemptedAt: params.CancelAttemptedAt.UTC().Format(time.RFC3339Nano), + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + if rowsAffected < 1 { + // No rows were updated, return the job as-is (already finalized) + return jobRowFromInternal(job) + } + + if params.ControlTopic != "" { + if err := dbsqlc.New().NotificationInsertJobCancel(ctx, dbtx, &dbsqlc.NotificationInsertJobCancelParams{ + JobID: job.ID, + Queue: job.Queue, + Topic: params.ControlTopic, + }); err != nil { + return nil, fmt.Errorf("error inserting job cancellation notification: %w", interpretError(err)) + } + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobCountByAllStates(ctx context.Context, params *riverdriver.JobCountByAllStatesParams) (map[rivertype.JobState]int, error) { + counts, err := dbsqlc.New().JobCountByAllStates(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + countsMap := make(map[rivertype.JobState]int) + for _, state := range rivertype.JobStates() { + countsMap[state] = 0 + } + for _, count := range counts { + countsMap[rivertype.JobState(count.State)] = int(count.Count) + } + return countsMap, nil +} + +func (e *Executor) JobCountByQueueAndState(ctx context.Context, params *riverdriver.JobCountByQueueAndStateParams) ([]*riverdriver.JobCountByQueueAndStateResult, error) { + rows, err := dbsqlc.New().JobCountByQueueAndState(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.QueueNames) + if err != nil { + return nil, interpretError(err) + } + + // MySQL's GROUP BY only returns queues that have jobs, so fill in zero + // counts for any requested queues not in the result. + countsByQueue := make(map[string]*riverdriver.JobCountByQueueAndStateResult, len(rows)) + for _, row := range rows { + countsByQueue[row.Queue] = &riverdriver.JobCountByQueueAndStateResult{ + CountAvailable: row.CountAvailable, + CountRunning: row.CountRunning, + Queue: row.Queue, + } + } + + var ( + queueNames = slices.Compact(slices.Sorted(slices.Values(params.QueueNames))) + results = make([]*riverdriver.JobCountByQueueAndStateResult, len(queueNames)) + ) + for i, name := range queueNames { + if result, ok := countsByQueue[name]; ok { + results[i] = result + } else { + results[i] = &riverdriver.JobCountByQueueAndStateResult{Queue: name} + } + } + + return results, nil +} + +func (e *Executor) JobCountByState(ctx context.Context, params *riverdriver.JobCountByStateParams) (int, error) { + numJobs, err := dbsqlc.New().JobCountByState(schemaTemplateParam(ctx, params.Schema), e.dbtx, string(params.State)) + if err != nil { + return 0, err + } + return int(numJobs), nil +} + +func (e *Executor) JobDelete(ctx context.Context, params *riverdriver.JobDeleteParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Lock the job while reading it so a concurrent producer can't change it + // to running between the read and delete statements. + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + res, err := dbsqlc.New().JobDeleteExec(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return nil, interpretError(err) + } + + if rowsAffected < 1 { + if rivertype.JobState(job.State) == rivertype.JobStateRunning { + return nil, rivertype.ErrJobRunning + } + return nil, fmt.Errorf("bug; expected only to fetch a job with state %q, but was: %q", rivertype.JobStateRunning, job.State) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobDeleteBefore(ctx context.Context, params *riverdriver.JobDeleteBeforeParams) (int, error) { + queuesExcluded := params.QueuesExcluded + if len(queuesExcluded) < 1 { + queuesExcluded = []string{""} + } + + queuesIncluded := params.QueuesIncluded + if len(queuesIncluded) < 1 { + queuesIncluded = []string{""} + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (int, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + // Do not bind zero-time literals for disabled retention policies: MySQL + // validates interpolated DATETIME constants even in a false predicate. + ids, err := dbsqlc.New().JobDeleteBeforeSelect(ctx, dbtx, &dbsqlc.JobDeleteBeforeSelectParams{ + CancelledDoDelete: boolToInt64(params.CancelledDoDelete), + CancelledFinalizedAtHorizon: sql.NullTime{Time: params.CancelledFinalizedAtHorizon.UTC(), Valid: params.CancelledDoDelete && !params.CancelledFinalizedAtHorizon.IsZero()}, + CompletedDoDelete: boolToInt64(params.CompletedDoDelete), + CompletedFinalizedAtHorizon: sql.NullTime{Time: params.CompletedFinalizedAtHorizon.UTC(), Valid: params.CompletedDoDelete && !params.CompletedFinalizedAtHorizon.IsZero()}, + DiscardedDoDelete: boolToInt64(params.DiscardedDoDelete), + DiscardedFinalizedAtHorizon: sql.NullTime{Time: params.DiscardedFinalizedAtHorizon.UTC(), Valid: params.DiscardedDoDelete && !params.DiscardedFinalizedAtHorizon.IsZero()}, + Limit: int32(params.Max), //nolint:gosec + QueuesExcluded: queuesExcluded, + QueuesExcludedEmpty: boolToInt64(len(params.QueuesExcluded) < 1), + QueuesIncluded: queuesIncluded, + QueuesIncludedEmpty: boolToInt64(params.QueuesIncluded == nil), + }) + if err != nil { + return 0, interpretError(err) + } + if len(ids) > 0 { + if err := dbsqlc.New().JobDeleteManyExec(ctx, dbtx, ids); err != nil { + return 0, interpretError(err) + } + } + return len(ids), nil + }) +} + +func (e *Executor) JobDeleteMany(ctx context.Context, params *riverdriver.JobDeleteManyParams) ([]*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*rivertype.JobRow, error) { + baseCtx := ctx + selectCtx := sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "order_by_clause": {Value: params.OrderByClause}, + "where_clause": {Value: params.WhereClause}, + }, params.NamedArgs) + selectCtx = schemaTemplateParam(selectCtx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Select and lock the rows before deleting them so that a producer can't + // start one of the selected jobs between the two statements. + jobs, err := dbsqlc.New().JobDeleteManySelect(selectCtx, dbtx, params.Max) + if err != nil { + return nil, interpretError(err) + } + + if len(jobs) > 0 { + // Use a fresh context with only the schema replacement. The select + // context has templates that aren't present in the DELETE query. + deleteCtx := schemaTemplateParam(baseCtx, params.Schema) + ids := sliceutil.Map(jobs, func(j *dbsqlc.RiverJob) int64 { return j.ID }) + if err := dbsqlc.New().JobDeleteManyExec(deleteCtx, dbtx, ids); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.MapError(jobs, jobRowFromInternal) + }) +} + +func (e *Executor) JobGetAvailable(ctx context.Context, params *riverdriver.JobGetAvailableParams) (*riverdriver.JobGetAvailableResult, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.JobGetAvailableResult, error) { + baseCtx := ctx + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + // Preserve byte-exact kind matching (including trailing spaces). sqlc + // loses slice arguments when the IN operand is a CAST expression. + selectCtx := sqlctemplate.WithReplacements(baseCtx, map[string]sqlctemplate.Replacement{ + "kind_column": {Stable: true, Value: "CAST(kind AS BINARY)"}, + }, nil) + selectCtx = schemaTemplateParam(selectCtx, params.Schema) + + claimedJobs, err := dbsqlc.New().JobGetAvailable(selectCtx, dbtx, &dbsqlc.JobGetAvailableParams{ + AttemptedBy: params.ClientID, + Kind: params.Kind, + KindFilter: boolToInt64(params.Kind != nil), + Limit: int32(params.MaxToLock), //nolint:gosec + MaxAttemptedBy: int64(params.MaxAttemptedBy), + Now: nullTimeFromPtr(params.Now), + Queue: params.Queue, + }) + if err != nil { + return nil, interpretError(err) + } + + if len(claimedJobs) == 0 { + return &riverdriver.JobGetAvailableResult{}, nil + } + + // The locking select projects each row's updated values. Use its server + // timestamp for the update as well, so the returned and stored rows agree. + if err := dbsqlc.New().JobGetAvailableUpdate(ctx, dbtx, &dbsqlc.JobGetAvailableUpdateParams{ + AttemptedBy: params.ClientID, + ID: sliceutil.Map(claimedJobs, func(job *dbsqlc.JobGetAvailableRow) int64 { return job.ID }), + MaxAttemptedBy: int64(params.MaxAttemptedBy), + Now: sql.NullTime{Time: claimedJobs[0].AttemptedAt, Valid: true}, + }); err != nil { + return nil, interpretError(err) + } + + jobs := sliceutil.Map(claimedJobs, func(job *dbsqlc.JobGetAvailableRow) *dbsqlc.RiverJob { + return &dbsqlc.RiverJob{ + ID: job.ID, + Args: job.Args, + Attempt: job.Attempt, + AttemptedAt: sql.NullTime{Time: job.AttemptedAt, Valid: true}, + AttemptedBy: job.AttemptedBy, + CreatedAt: job.CreatedAt, + Errors: job.Errors, + FinalizedAt: job.FinalizedAt, + Kind: job.Kind, + MaxAttempts: job.MaxAttempts, + Metadata: job.Metadata, + Priority: job.Priority, + Queue: job.Queue, + State: string(rivertype.JobStateRunning), + ScheduledAt: job.ScheduledAt, + Tags: job.Tags, + UniqueKey: job.UniqueKey, + UniqueStates: job.UniqueStates, + } + }) + + return jobGetAvailableResultFromInternal(jobs), nil + }) +} + +func (e *Executor) JobGetByID(ctx context.Context, params *riverdriver.JobGetByIDParams) (*rivertype.JobRow, error) { + job, err := dbsqlc.New().JobGetByID(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) +} + +func (e *Executor) JobGetByIDMany(ctx context.Context, params *riverdriver.JobGetByIDManyParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetByIDMany(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobGetByKindMany(ctx context.Context, params *riverdriver.JobGetByKindManyParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetByKindMany(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Kind) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobGetCancelRequested(ctx context.Context, params *riverdriver.JobGetCancelRequestedParams) ([]int64, error) { + ids, err := dbsqlc.New().JobGetCancelRequested(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.ID) + return ids, interpretError(err) +} + +func (e *Executor) JobGetStuck(ctx context.Context, params *riverdriver.JobGetStuckParams) ([]*rivertype.JobRow, error) { + jobs, err := dbsqlc.New().JobGetStuck(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.JobGetStuckParams{ + AfterID: params.AfterID, + Limit: int32(params.Max), //nolint:gosec + StuckHorizon: sql.NullTime{Time: params.StuckHorizon.UTC(), Valid: true}, + }) + if err != nil { + return nil, interpretError(err) + } + return jobRowsFromInternalPartial(jobs), nil +} + +func (e *Executor) JobInsertFastMany(ctx context.Context, params *riverdriver.JobInsertFastManyParams) ([]*riverdriver.JobInsertFastResult, error) { + // Match Postgres's rejection of a batch that affects one unique row twice. + uniqueKeysInBatch := make(map[string]bool, len(params.Jobs)) + for _, job := range params.Jobs { + if len(job.UniqueKey) > 0 && job.UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{job.State}) != 0 { + key := string(job.UniqueKey) + if uniqueKeysInBatch[key] { + return nil, errors.New("unique key appears more than once in batch") + } + uniqueKeysInBatch[key] = true + } + } + + var ( + insertRes = make([]*riverdriver.JobInsertFastResult, len(params.Jobs)) + uniqueNonceByIndex = make([]string, len(params.Jobs)) + ) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + baseCtx := ctx + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + var ( + idsWithNonce []int64 + jobs []*dbsqlc.RiverJob + ) + if jobInsertBatchSupported(params) { + var err error + jobs, err = jobInsertBatchReturning(baseCtx, dbtx, params, uniqueNonceByIndex) + if err != nil { + return err + } + idsWithNonce = sliceutil.Map(jobs, func(job *dbsqlc.RiverJob) int64 { return job.ID }) + } else { + plainInsertCtx := schemaTemplateParam(baseCtx, params.Schema) + plainInsertCtx = sqlctemplate.WithReplacements(plainInsertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: ""}, + }, nil) + uniqueInsertCtx := schemaTemplateParam(baseCtx, params.Schema) + uniqueInsertCtx = sqlctemplate.WithReplacements(uniqueInsertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: "ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id)"}, + }, nil) + + ids := make([]int64, len(params.Jobs)) + for i, jobParams := range params.Jobs { + // Only use MySQL's broad ON DUPLICATE KEY behavior when a River unique + // key is present. Otherwise, a primary key conflict or a user-added + // unique constraint must be returned as an error. + insertCtx := plainInsertCtx + if len(jobParams.UniqueKey) > 0 { + insertCtx = uniqueInsertCtx + + // Postgres can inspect xmax to determine whether an upsert inserted a + // row. MySQL has no equivalent, so add a per-row nonce to metadata and + // compare it with the selected result. Each input needs its own nonce + // so duplicates within this batch are classified correctly. + uniqueNonceByIndex[i] = randutil.Hex(8) + } + + insertParams, err := jobInsertFastParams(jobParams, uniqueNonceByIndex[i]) + if err != nil { + return err + } + + res, err := dbsqlc.New().JobInsertFast(insertCtx, dbtx, insertParams) + if err != nil { + return interpretError(err) + } + ids[i], err = res.LastInsertId() + if err != nil { + return err + } + if uniqueNonceByIndex[i] != "" { + idsWithNonce = append(idsWithNonce, ids[i]) + } + } + + selected, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, ids) + if err != nil { + return interpretError(err) + } + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(selected)) + for _, job := range selected { + jobsByID[job.ID] = job + } + jobs = sliceutil.Map(ids, func(id int64) *dbsqlc.RiverJob { return jobsByID[id] }) + } + + for i, internal := range jobs { + job, err := jobRowFromInternal(internal) + if err != nil { + return err + } + + insertRes[i] = &riverdriver.JobInsertFastResult{Job: job} + if uniqueNonceByIndex[i] != "" { + uniqueSkippedAsDuplicate := gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).Str != uniqueNonceByIndex[i] + if uniqueSkippedAsDuplicate && (!bytes.Equal(job.UniqueKey, params.Jobs[i].UniqueKey) || + !slices.Contains(job.UniqueStates, job.State) || + params.Jobs[i].UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{params.Jobs[i].State}) == 0) { + // Unlike Postgres and SQLite, MySQL's ON DUPLICATE KEY + // cannot target River's unique-job index. Reject a conflict + // on any other unique constraint instead of silently treating + // the unrelated row as the requested unique job. + return fmt.Errorf("job insert at index %d conflicted on a non-River unique constraint", i) + } + insertRes[i].UniqueSkippedAsDuplicate = uniqueSkippedAsDuplicate + job.Metadata, err = sjson.DeleteBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey) + if err != nil { + return fmt.Errorf("error removing unique nonce from returned job metadata: %w", err) + } + } + } + + if len(idsWithNonce) > 0 { + if err := dbsqlc.New().JobClearUniqueNonce(ctx, dbtx, idsWithNonce); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return nil, err + } + + return insertRes, nil +} + +func (e *Executor) JobInsertFastManyNoReturning(ctx context.Context, params *riverdriver.JobInsertFastManyParams) (int, error) { + // RowsAffected can't distinguish a no-op duplicate from an insert when the + // DSN enables clientFoundRows. Fall back to nonce-based duplicate detection + // for any batch where a conflict is possible. + if slices.ContainsFunc(params.Jobs, func(job *riverdriver.JobInsertFastParams) bool { + return len(job.UniqueKey) > 0 + }) { + results, err := e.JobInsertFastMany(ctx, params) + if err != nil { + return 0, err + } + + numInserted := 0 + for _, result := range results { + if !result.UniqueSkippedAsDuplicate { + numInserted++ + } + } + return numInserted, nil + } + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + baseCtx := ctx + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + insertCtx := schemaTemplateParam(baseCtx, params.Schema) + if len(params.Jobs) > 1 { + _, err := jobInsertBatch(insertCtx, dbtx, params, nil) + return err + } + insertCtx = sqlctemplate.WithReplacements(insertCtx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Value: ""}, + }, nil) + + for _, jobParams := range params.Jobs { + insertParams, err := jobInsertFastParams(jobParams, "") + if err != nil { + return err + } + + if _, err := dbsqlc.New().JobInsertFast(insertCtx, dbtx, insertParams); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return 0, err + } + + return len(params.Jobs), nil +} + +// jobInsertFastParams builds the common insert parameters for a single job. +// If uniqueNonce is non-empty, it's set in the job's metadata for duplicate +// detection. +func jobInsertFastParams(params *riverdriver.JobInsertFastParams, uniqueNonce string) (*dbsqlc.JobInsertFastParams, error) { + metadata := sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")) + if uniqueNonce != "" { + var err error + metadata, err = sjson.SetBytes(metadata, riverdriver.UniqueInsertMetadataKey, uniqueNonce) + if err != nil { + return nil, fmt.Errorf("error adding unique nonce to job metadata: %w", err) + } + } + + tags, err := json.Marshal(params.Tags) + if err != nil { + return nil, err + } + + var uniqueStates sql.NullInt16 + if params.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(params.UniqueStates), Valid: true} + } + + var id sql.NullInt64 + if params.ID != nil { + id = sql.NullInt64{Int64: *params.ID, Valid: true} + } + + return &dbsqlc.JobInsertFastParams{ + ID: id, + Args: params.EncodedArgs, + CreatedAt: utcTimePtr(params.CreatedAt), + Kind: params.Kind, + MaxAttempts: int64(params.MaxAttempts), + Metadata: metadata, + Priority: int16(params.Priority), //nolint:gosec + Queue: params.Queue, + ScheduledAt: utcTimePtr(params.ScheduledAt), + State: string(params.State), + Tags: tags, + UniqueKey: nullStringFromBytes(params.UniqueKey), + UniqueStates: uniqueStates, + }, nil +} + +func (e *Executor) JobInsertFull(ctx context.Context, params *riverdriver.JobInsertFullParams) (*rivertype.JobRow, error) { + var attemptedBy []byte + if params.AttemptedBy != nil { + var err error + attemptedBy, err = json.Marshal(params.AttemptedBy) + if err != nil { + return nil, err + } + } + + var errorsData []byte + if len(params.Errors) > 0 { + var err error + errorsData, err = json.Marshal(sliceutil.Map(params.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return nil, err + } + } + + tags, err := json.Marshal(params.Tags) + if err != nil { + return nil, err + } + + var uniqueStates sql.NullInt16 + if params.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(params.UniqueStates), Valid: true} + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + lastInsertID, err := dbsqlc.New().JobInsertFullExec(ctx, dbtx, &dbsqlc.JobInsertFullExecParams{ + Attempt: int64(params.Attempt), + AttemptedAt: nullTimeFromPtr(params.AttemptedAt), + AttemptedBy: attemptedBy, + Args: params.EncodedArgs, + CreatedAt: utcTimePtr(params.CreatedAt), + Errors: errorsData, + FinalizedAt: nullTimeFromPtr(params.FinalizedAt), + Kind: params.Kind, + MaxAttempts: int64(params.MaxAttempts), + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + Priority: int16(params.Priority), //nolint:gosec + Queue: params.Queue, + ScheduledAt: utcTimePtr(params.ScheduledAt), + State: string(params.State), + Tags: tags, + UniqueKey: nullStringFromBytes(params.UniqueKey), + UniqueStates: uniqueStates, + }) + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, lastInsertID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobInsertFullMany(ctx context.Context, params *riverdriver.JobInsertFullManyParams) ([]*rivertype.JobRow, error) { + insertRes := make([]*rivertype.JobRow, len(params.Jobs)) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + ids := make([]int64, len(params.Jobs)) + + for i, jobParams := range params.Jobs { + var attemptedBy []byte + if jobParams.AttemptedBy != nil { + var err error + attemptedBy, err = json.Marshal(jobParams.AttemptedBy) + if err != nil { + return err + } + } + + var errorsData []byte + if len(jobParams.Errors) > 0 { + var err error + errorsData, err = json.Marshal(sliceutil.Map(jobParams.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return err + } + } + + tags, err := json.Marshal(jobParams.Tags) + if err != nil { + return err + } + + var uniqueStates sql.NullInt16 + if jobParams.UniqueStates != 0 { + uniqueStates = sql.NullInt16{Int16: int16(jobParams.UniqueStates), Valid: true} + } + + ids[i], err = dbsqlc.New().JobInsertFullExec(ctx, dbtx, &dbsqlc.JobInsertFullExecParams{ + Attempt: int64(jobParams.Attempt), + AttemptedAt: nullTimeFromPtr(jobParams.AttemptedAt), + AttemptedBy: attemptedBy, + Args: jobParams.EncodedArgs, + CreatedAt: utcTimePtr(jobParams.CreatedAt), + Errors: errorsData, + FinalizedAt: nullTimeFromPtr(jobParams.FinalizedAt), + Kind: jobParams.Kind, + MaxAttempts: int64(jobParams.MaxAttempts), + Metadata: sliceutil.FirstNonEmpty(jobParams.Metadata, []byte("{}")), + Priority: int16(jobParams.Priority), //nolint:gosec + Queue: jobParams.Queue, + ScheduledAt: utcTimePtr(jobParams.ScheduledAt), + State: string(jobParams.State), + Tags: tags, + UniqueKey: nullStringFromBytes(jobParams.UniqueKey), + UniqueStates: uniqueStates, + }) + if err != nil { + return interpretError(err) + } + } + + jobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, ids) + if err != nil { + return interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(jobs)) + for _, j := range jobs { + jobsByID[j.ID] = j + } + + for i, id := range ids { + insertRes[i], err = jobRowFromInternal(jobsByID[id]) + if err != nil { + return err + } + } + + return nil + }); err != nil { + return nil, err + } + + return insertRes, nil +} + +func (e *Executor) JobKindList(ctx context.Context, params *riverdriver.JobKindListParams) ([]string, error) { + exclude := params.Exclude + if len(exclude) == 0 { + exclude = []string{""} + } + + kinds, err := dbsqlc.New().JobKindList(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.JobKindListParams{ + After: params.After, + Exclude: exclude, + Match: params.Match, + Limit: int32(min(params.Max, math.MaxInt32)), //nolint:gosec + }) + if err != nil { + return nil, interpretError(err) + } + return kinds, nil +} + +func (e *Executor) JobList(ctx context.Context, params *riverdriver.JobListParams) ([]*rivertype.JobRow, error) { + ctx = sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "order_by_clause": {Value: params.OrderByClause}, + "where_clause": {Value: params.WhereClause}, + }, params.NamedArgs) + + jobs, err := dbsqlc.New().JobList(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Max) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.MapError(jobs, jobRowFromInternal) +} + +func (e *Executor) JobRescueMany(ctx context.Context, params *riverdriver.JobRescueManyParams) (*struct{}, error) { + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if len(params.ID) > 1 && batchIDsUnique(params.ID) { + if err := jobRescueBatch(ctx, dbtx, params); !isNoDatabaseSelected(err) { + return err + } + } + + for i := range params.ID { + if err := dbsqlc.New().JobRescue(ctx, dbtx, &dbsqlc.JobRescueParams{ + ID: params.ID[i], + Error: params.Error[i], + FinalizedAt: nullTimeFromPtr(params.FinalizedAt[i]), + ScheduledAt: params.ScheduledAt[i].UTC(), + State: params.State[i], + StuckHorizon: sql.NullTime{Time: params.StuckHorizon.UTC(), Valid: true}, + }); err != nil { + return interpretError(err) + } + } + + return nil + }); err != nil { + return nil, err + } + + return &struct{}{}, nil +} + +func (e *Executor) JobRetry(ctx context.Context, params *riverdriver.JobRetryParams) (*rivertype.JobRow, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + _, err := dbsqlc.New().JobRetryExec(ctx, dbtx, &dbsqlc.JobRetryExecParams{ + ID: params.ID, + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobSchedule(ctx context.Context, params *riverdriver.JobScheduleParams) ([]*riverdriver.JobScheduleResult, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.JobScheduleResult, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + scheduleResults, err := dbsqlc.New().JobSchedule(ctx, dbtx, &dbsqlc.JobScheduleParams{ + Limit: int32(params.Max), //nolint:gosec + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + + var ( + allIDs []int64 + availIDs []int64 + discardIDs []int64 + discardSet = make(map[int64]bool) + ) + + for _, result := range scheduleResults { + allIDs = append(allIDs, result.ID) + if result.ConflictDiscarded != 0 { + discardIDs = append(discardIDs, result.ID) + discardSet[result.ID] = true + } else { + availIDs = append(availIDs, result.ID) + } + } + + if len(availIDs) > 0 { + if err := dbsqlc.New().JobScheduleSetAvailableExec(ctx, dbtx, availIDs); err != nil { + return nil, interpretError(err) + } + } + + if len(discardIDs) > 0 { + if err := dbsqlc.New().JobScheduleSetDiscardedExec(ctx, dbtx, &dbsqlc.JobScheduleSetDiscardedExecParams{ + ID: discardIDs, + Now: nullTimeFromPtr(params.Now), + }); err != nil { + return nil, interpretError(err) + } + } + + if len(allIDs) == 0 { + return nil, nil + } + + updatedJobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, allIDs) + if err != nil { + return nil, interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(updatedJobs)) + for _, j := range updatedJobs { + jobsByID[j.ID] = j + } + + // Return results in the same order as scheduleResults. + results := make([]*riverdriver.JobScheduleResult, len(scheduleResults)) + for i, sr := range scheduleResults { + job, _ := jobRowFromInternalPartial(jobsByID[sr.ID]) + results[i] = &riverdriver.JobScheduleResult{ConflictDiscarded: discardSet[sr.ID], Job: *job} + } + + return results, nil + }) +} + +func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdriver.JobSetStateIfRunningManyParams) ([]*rivertype.JobRow, error) { + setRes := make([]*rivertype.JobRow, 0, len(params.ID)) + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + metadata, err := jobMetadataMergeMany(ctx, dbtx, params.ID, params.MetadataDoMerge, params.MetadataUpdates) + if err != nil { + return err + } + paramsCopy := *params + paramsCopy.MetadataUpdates = metadata + params := ¶msCopy + + batchDone := false + if len(params.ID) > 1 && batchIDsUnique(params.ID) { + err := jobSetStateBatch(ctx, dbtx, params) + if err != nil && !isNoDatabaseSelected(err) { + return fmt.Errorf("error setting job state: %w", err) + } + batchDone = err == nil + } + if !batchDone { + // Execute all state changes, including metadata-only updates to non-running jobs. + for i := range params.ID { + setStateParams := &dbsqlc.JobSetStateIfRunningExecParams{ + ID: params.ID[i], + Error: []byte("{}"), + MetadataUpdates: []byte("{}"), + Now: nullTimeFromPtr(params.Now), + State: string(params.State[i]), + } + + if params.Attempt[i] != nil { + setStateParams.AttemptDoUpdate = 1 + setStateParams.Attempt = int64(*params.Attempt[i]) + } + if params.ErrData[i] != nil { + setStateParams.ErrorsDoUpdate = 1 + setStateParams.Error = params.ErrData[i] + } + if params.FinalizedAt[i] != nil { + setStateParams.FinalizedAtDoUpdate = 1 + setStateParams.FinalizedAt = nullTimeFromPtr(params.FinalizedAt[i]) + } + if params.MetadataDoMerge[i] { + setStateParams.MetadataDoMerge = 1 + setStateParams.MetadataUpdates = params.MetadataUpdates[i] + } + if params.ScheduledAt[i] != nil { + setStateParams.ScheduledAtDoUpdate = 1 + setStateParams.ScheduledAt = params.ScheduledAt[i].UTC() + } + + if err := dbsqlc.New().JobSetStateIfRunningExec(ctx, dbtx, setStateParams); err != nil { + return fmt.Errorf("error setting job state: %w", err) + } + } + } + + // Use a current read to include jobs that another transaction already finalized. + jobs, err := dbsqlc.New().JobGetByIDManyForUpdate(ctx, dbtx, params.ID) + if err != nil { + return interpretError(err) + } + + jobsByID := make(map[int64]*dbsqlc.RiverJob, len(jobs)) + for _, j := range jobs { + jobsByID[j.ID] = j + } + + // Build results in original order. + for _, id := range params.ID { + job := jobsByID[id] + if job == nil { + continue + } + + jobRow, _ := jobRowFromInternalPartial(job) + setRes = append(setRes, jobRow) + } + + return nil + }); err != nil { + return nil, err + } + + return setRes, nil +} + +func (e *Executor) JobUpdate(ctx context.Context, params *riverdriver.JobUpdateParams) (*rivertype.JobRow, error) { + metadata := params.Metadata + if metadata == nil { + metadata = []byte("{}") + } + + var metadataDoMerge int64 + if params.MetadataDoMerge { + metadataDoMerge = 1 + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + merged, err := jobMetadataMergeMany(ctx, dbtx, []int64{params.ID}, []bool{params.MetadataDoMerge}, [][]byte{metadata}) + if err != nil { + return nil, err + } + if err := dbsqlc.New().JobUpdateExec(ctx, dbtx, &dbsqlc.JobUpdateExecParams{ + ID: params.ID, + MetadataDoMerge: metadataDoMerge, + Metadata: merged[0], + }); err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) JobUpdateFull(ctx context.Context, params *riverdriver.JobUpdateFullParams) (*rivertype.JobRow, error) { + attemptedAt := params.AttemptedAt + if attemptedAt != nil { + attemptedAt = new(attemptedAt.UTC()) + } + + attemptedBy, err := json.Marshal(params.AttemptedBy) + if err != nil { + return nil, err + } + + errorsData, err := json.Marshal(sliceutil.Map(params.Errors, func(e []byte) json.RawMessage { return json.RawMessage(e) })) + if err != nil { + return nil, err + } + + finalizedAt := params.FinalizedAt + if finalizedAt != nil { + finalizedAt = new(finalizedAt.UTC()) + } + + metadata := params.Metadata + if metadata == nil { + metadata = []byte("{}") + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.JobRow, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().JobUpdateFullExec(ctx, dbtx, &dbsqlc.JobUpdateFullExecParams{ + ID: params.ID, + Attempt: int64(params.Attempt), + AttemptDoUpdate: boolToInt64(params.AttemptDoUpdate), + AttemptedAt: nullTimeFromPtr(attemptedAt), + AttemptedAtDoUpdate: boolToInt64(params.AttemptedAtDoUpdate), + AttemptedBy: attemptedBy, + AttemptedByDoUpdate: boolToInt64(params.AttemptedByDoUpdate), + ErrorsDoUpdate: boolToInt64(params.ErrorsDoUpdate), + Errors: errorsData, + FinalizedAtDoUpdate: boolToInt64(params.FinalizedAtDoUpdate), + FinalizedAt: nullTimeFromPtr(finalizedAt), + MaxAttemptsDoUpdate: boolToInt64(params.MaxAttemptsDoUpdate), + MaxAttempts: int64(min(params.MaxAttempts, math.MaxInt64)), + MetadataDoUpdate: boolToInt64(params.MetadataDoUpdate), + Metadata: metadata, + StateDoUpdate: boolToInt64(params.StateDoUpdate), + State: string(params.State), + }); err != nil { + return nil, interpretError(err) + } + + job, err := dbsqlc.New().JobGetByIDForUpdate(ctx, dbtx, params.ID) + if err != nil { + return nil, interpretError(err) + } + + return jobRowFromInternal(job) + }) +} + +func (e *Executor) LeaderAttemptElect(ctx context.Context, params *riverdriver.LeaderElectParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + err := dbsqlc.New().LeaderAttemptElectExec(ctx, dbtx, &dbsqlc.LeaderAttemptElectExecParams{ + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + TTL: params.TTL.Microseconds(), + }) + if err != nil { + if isDuplicateEntry(err) { + return nil, rivertype.ErrNotFound + } + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetElectedLeader(ctx, dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderAttemptReelect(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + err := dbsqlc.New().LeaderAttemptReelectExec(ctx, dbtx, &dbsqlc.LeaderAttemptReelectExecParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + TTL: params.TTL.Microseconds(), + }) + if err != nil { + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetReelected(ctx, dbtx, &dbsqlc.LeaderGetReelectedParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + Now: nullTimeFromPtr(params.Now), + }) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderDeleteExpired(ctx context.Context, params *riverdriver.LeaderDeleteExpiredParams) (int, error) { + numDeleted, err := dbsqlc.New().LeaderDeleteExpired(schemaTemplateParam(ctx, params.Schema), e.dbtx, nullTimeFromPtr(params.Now)) + if err != nil { + return 0, interpretError(err) + } + return int(numDeleted), nil +} + +func (e *Executor) LeaderGetElectedLeader(ctx context.Context, params *riverdriver.LeaderGetElectedLeaderParams) (*riverdriver.Leader, error) { + leader, err := dbsqlc.New().LeaderGetElectedLeader(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil +} + +func (e *Executor) LeaderInsert(ctx context.Context, params *riverdriver.LeaderInsertParams) (*riverdriver.Leader, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*riverdriver.Leader, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().LeaderInsertExec(ctx, dbtx, &dbsqlc.LeaderInsertExecParams{ + ElectedAt: utcTimePtr(params.ElectedAt), + ExpiresAt: utcTimePtr(params.ExpiresAt), + Now: nullTimeFromPtr(params.Now), + LeaderID: params.LeaderID, + TTL: params.TTL.Microseconds(), + }); err != nil { + return nil, interpretError(err) + } + + leader, err := dbsqlc.New().LeaderGetElectedLeader(ctx, dbtx) + if err != nil { + return nil, interpretError(err) + } + return leaderFromInternal(leader), nil + }) +} + +func (e *Executor) LeaderResign(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) { + numResigned, err := dbsqlc.New().LeaderResign(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.LeaderResignParams{ + ElectedAt: params.ElectedAt.UTC(), + LeaderID: params.LeaderID, + }) + if err != nil { + return false, interpretError(err) + } + return numResigned > 0, nil +} + +func (e *Executor) MigrationDeleteAssumingMainMany(ctx context.Context, params *riverdriver.MigrationDeleteAssumingMainManyParams) ([]*riverdriver.Migration, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.Migration, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + migrations, err := dbsqlc.New().RiverMigrationDeleteAssumingMainMany(ctx, dbtx, versions) + if err != nil { + return nil, interpretError(err) + } + + if len(versions) > 0 { + if err := dbsqlc.New().RiverMigrationDeleteAssumingMainManyExec(ctx, dbtx, versions); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.Map(migrations, func(internal *dbsqlc.RiverMigrationDeleteAssumingMainManyRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }), nil + }) +} + +func (e *Executor) MigrationDeleteByLineAndVersionMany(ctx context.Context, params *riverdriver.MigrationDeleteByLineAndVersionManyParams) ([]*riverdriver.Migration, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]*riverdriver.Migration, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + migrations, err := dbsqlc.New().RiverMigrationDeleteByLineAndVersionMany(ctx, dbtx, &dbsqlc.RiverMigrationDeleteByLineAndVersionManyParams{ + Line: params.Line, + Version: versions, + }) + if err != nil { + return nil, interpretError(err) + } + + if len(versions) > 0 { + if err := dbsqlc.New().RiverMigrationDeleteByLineAndVersionManyExec(ctx, dbtx, &dbsqlc.RiverMigrationDeleteByLineAndVersionManyExecParams{ + Line: params.Line, + Version: versions, + }); err != nil { + return nil, interpretError(err) + } + } + + return sliceutil.Map(migrations, migrationFromInternal), nil + }) +} + +func (e *Executor) MigrationGetAllAssumingMain(ctx context.Context, params *riverdriver.MigrationGetAllAssumingMainParams) ([]*riverdriver.Migration, error) { + migrations, err := dbsqlc.New().RiverMigrationGetAllAssumingMain(schemaTemplateParam(ctx, params.Schema), e.dbtx) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(migrations, func(internal *dbsqlc.RiverMigrationGetAllAssumingMainRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }), nil +} + +func (e *Executor) MigrationGetByLine(ctx context.Context, params *riverdriver.MigrationGetByLineParams) ([]*riverdriver.Migration, error) { + migrations, err := dbsqlc.New().RiverMigrationGetByLine(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Line) + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(migrations, migrationFromInternal), nil +} + +func (e *Executor) MigrationInsertMany(ctx context.Context, params *riverdriver.MigrationInsertManyParams) ([]*riverdriver.Migration, error) { + var migrations []*riverdriver.Migration + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + for _, version := range params.Versions { + if err := dbsqlc.New().RiverMigrationInsertExec(ctx, dbtx, &dbsqlc.RiverMigrationInsertExecParams{ + Line: params.Line, + Version: int64(version), + }); err != nil { + return interpretError(err) + } + } + + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + internals, err := dbsqlc.New().RiverMigrationGetByLineAndVersionMany(ctx, dbtx, &dbsqlc.RiverMigrationGetByLineAndVersionManyParams{ + Line: params.Line, + Version: versions, + }) + if err != nil { + return interpretError(err) + } + + migrations = sliceutil.Map(internals, migrationFromInternal) + return nil + }); err != nil { + return nil, err + } + + return migrations, nil +} + +func (e *Executor) MigrationInsertManyAssumingMain(ctx context.Context, params *riverdriver.MigrationInsertManyAssumingMainParams) ([]*riverdriver.Migration, error) { + var migrations []*riverdriver.Migration + + if err := dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + for _, version := range params.Versions { + if err := dbsqlc.New().RiverMigrationInsertAssumingMainExec(ctx, dbtx, int64(version)); err != nil { + return interpretError(err) + } + } + + versions := sliceutil.Map(params.Versions, func(v int) int64 { return int64(v) }) + + internals, err := dbsqlc.New().RiverMigrationGetByVersionMany(ctx, dbtx, versions) + if err != nil { + return interpretError(err) + } + + migrations = sliceutil.Map(internals, func(internal *dbsqlc.RiverMigrationGetByVersionManyRow) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: riverdriver.MigrationLineMain, + Version: int(internal.Version), + } + }) + return nil + }); err != nil { + return nil, err + } + + return migrations, nil +} + +func (e *Executor) NotificationDeleteBefore(ctx context.Context, params *riverdriver.NotificationDeleteBeforeParams) (int, error) { + numDeleted, err := dbsqlc.New().NotificationDeleteBefore(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.NotificationDeleteBeforeParams{CreatedAtHorizon: params.CreatedAtHorizon.UTC(), Limit: int32(params.Max)}) //nolint:gosec + return int(numDeleted), interpretError(err) +} + +func (e *Executor) NotifyMany(ctx context.Context, params *riverdriver.NotifyManyParams) error { + if len(params.Payload) < 1 { + return nil + } + + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if len(params.Payload) > 1 { + return execJSONBatch(ctx, dbtx, notificationInsertBatchSQL, params.Payload, params.Topic) + } + + for _, payload := range params.Payload { + if err := dbsqlc.New().NotificationInsert(ctx, dbtx, &dbsqlc.NotificationInsertParams{ + Payload: payload, + Topic: params.Topic, + }); err != nil { + return interpretError(err) + } + } + + return nil + }) +} + +func (e *Executor) Ping(ctx context.Context) error { + _, err := dbsqlc.New().Ping(ctx, e.dbtx) + return interpretError(err) +} + +func (e *Executor) PGAdvisoryXactLock(ctx context.Context, key int64) (*struct{}, error) { + // MySQL has GET_LOCK which is similar to Postgres advisory locks, but + // it's session-scoped rather than transaction-scoped. For now, return + // not implemented. + return nil, riverdriver.ErrNotImplemented +} + +func (e *Executor) QueueCreateOrSetUpdatedAt(ctx context.Context, params *riverdriver.QueueCreateOrSetUpdatedAtParams) (*rivertype.Queue, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.Queue, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().QueueCreateOrSetUpdatedAtExec(ctx, dbtx, &dbsqlc.QueueCreateOrSetUpdatedAtExecParams{ + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + Name: params.Name, + Now: nullTimeFromPtr(params.Now), + PausedAt: nullTimeFromPtr(params.PausedAt), + UpdatedAt: nullTimeFromPtr(params.UpdatedAt), + }); err != nil { + return nil, interpretError(err) + } + + queue, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil + }) +} + +func (e *Executor) QueueDeleteExpired(ctx context.Context, params *riverdriver.QueueDeleteExpiredParams) ([]string, error) { + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) ([]string, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + queueNames, err := dbsqlc.New().QueueDeleteExpiredSelect(ctx, dbtx, &dbsqlc.QueueDeleteExpiredSelectParams{ + Limit: int32(params.Max), //nolint:gosec + UpdatedAtHorizon: params.UpdatedAtHorizon.UTC(), + }) + if err != nil { + return nil, interpretError(err) + } + + if len(queueNames) > 0 { + if err := dbsqlc.New().QueueDeleteExpiredExec(ctx, dbtx, queueNames); err != nil { + return nil, interpretError(err) + } + } + + return queueNames, nil + }) +} + +func (e *Executor) QueueGet(ctx context.Context, params *riverdriver.QueueGetParams) (*rivertype.Queue, error) { + queue, err := dbsqlc.New().QueueGet(schemaTemplateParam(ctx, params.Schema), e.dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil +} + +func (e *Executor) QueueList(ctx context.Context, params *riverdriver.QueueListParams) ([]*rivertype.Queue, error) { + queues, err := dbsqlc.New().QueueList(schemaTemplateParam(ctx, params.Schema), e.dbtx, int32(params.Max)) //nolint:gosec + if err != nil { + return nil, interpretError(err) + } + return sliceutil.Map(queues, queueFromInternal), nil +} + +func (e *Executor) QueueNameList(ctx context.Context, params *riverdriver.QueueNameListParams) ([]string, error) { + exclude := params.Exclude + if len(exclude) == 0 { + exclude = []string{""} + } + queueNames, err := dbsqlc.New().QueueNameList(schemaTemplateParam(ctx, params.Schema), e.dbtx, &dbsqlc.QueueNameListParams{ + After: params.After, + Exclude: exclude, + Match: params.Match, + Limit: int32(min(params.Max, math.MaxInt32)), //nolint:gosec + }) + if err != nil { + return nil, interpretError(err) + } + return queueNames, nil +} + +func (e *Executor) QueuePause(ctx context.Context, params *riverdriver.QueuePauseParams) error { + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + now := nullTimeFromPtr(params.Now) + + var ( + res sql.Result + err error + ) + + if params.Name == riverdriver.AllQueuesString { + res, err = dbsqlc.New().QueuePauseAll(ctx, dbtx, &dbsqlc.QueuePauseAllParams{Now: now}) + } else { + res, err = dbsqlc.New().QueuePauseByName(ctx, dbtx, &dbsqlc.QueuePauseByNameParams{ + Name: params.Name, + Now: now, + }) + } + if err != nil { + return interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return interpretError(err) + } + + // MySQL only reports rows that actually changed values, not rows + // matched. If no rows were affected for a named queue, check if the + // queue exists before returning ErrNotFound (it may already be paused). + if rowsAffected < 1 && params.Name != riverdriver.AllQueuesString { + if _, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name); err != nil { + return interpretError(err) + } + } + return nil + }) +} + +func (e *Executor) QueueResume(ctx context.Context, params *riverdriver.QueueResumeParams) error { + return dbutil.WithTx(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) error { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + now := nullTimeFromPtr(params.Now) + + var ( + res sql.Result + err error + ) + + if params.Name == riverdriver.AllQueuesString { + res, err = dbsqlc.New().QueueResumeAll(ctx, dbtx, now) + } else { + res, err = dbsqlc.New().QueueResumeByName(ctx, dbtx, &dbsqlc.QueueResumeByNameParams{ + Name: params.Name, + Now: now, + }) + } + if err != nil { + return interpretError(err) + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return interpretError(err) + } + + // MySQL only reports rows that actually changed values, not rows + // matched. If no rows were affected for a named queue, check if the + // queue exists before returning ErrNotFound (it may already be resumed). + if rowsAffected < 1 && params.Name != riverdriver.AllQueuesString { + if _, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name); err != nil { + return interpretError(err) + } + } + return nil + }) +} + +func (e *Executor) QueueUpdate(ctx context.Context, params *riverdriver.QueueUpdateParams) (*rivertype.Queue, error) { + var metadataDoUpdate int64 + if params.MetadataDoUpdate { + metadataDoUpdate = 1 + } + + return dbutil.WithTxV(ctx, e.executor(), func(ctx context.Context, execTx riverdriver.ExecutorTx) (*rivertype.Queue, error) { + ctx = schemaTemplateParam(ctx, params.Schema) + dbtx := templateReplaceWrapper{dbtx: e.driver.UnwrapTx(execTx), replacer: &e.driver.replacer} + + if err := dbsqlc.New().QueueUpdateExec(ctx, dbtx, &dbsqlc.QueueUpdateExecParams{ + Metadata: sliceutil.FirstNonEmpty(params.Metadata, []byte("{}")), + MetadataDoUpdate: metadataDoUpdate, + Name: params.Name, + }); err != nil { + return nil, interpretError(err) + } + + queue, err := dbsqlc.New().QueueGetForUpdate(ctx, dbtx, params.Name) + if err != nil { + return nil, interpretError(err) + } + return queueFromInternal(queue), nil + }) +} + +func (e *Executor) QueryRow(ctx context.Context, sql string, args ...any) riverdriver.Row { + return e.dbtx.QueryRowContext(ctx, sql, args...) +} + +func (e *Executor) SchemaCreate(ctx context.Context, params *riverdriver.SchemaCreateParams) error { + // In MySQL, schemas are databases. Create one if it doesn't exist. + if params.Schema != "" { + _, err := e.dbtx.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+mysqlIdentifier(params.Schema)) + return interpretError(err) + } + return nil +} + +func (e *Executor) SchemaDrop(ctx context.Context, params *riverdriver.SchemaDropParams) error { + if params.Schema != "" { + _, err := e.dbtx.ExecContext(ctx, "DROP DATABASE IF EXISTS "+mysqlIdentifier(params.Schema)) + return interpretError(err) + } + return nil +} + +func (e *Executor) SchemaGetExpired(ctx context.Context, params *riverdriver.SchemaGetExpiredParams) ([]string, error) { + schemas, err := dbsqlc.New().SchemaGetExpired(informationSchemaParam(ctx), e.dbtx, &dbsqlc.SchemaGetExpiredParams{ + BeforeName: params.BeforeName, + Prefix: params.Prefix, + }) + return schemas, interpretError(err) +} + +func (e *Executor) TableExists(ctx context.Context, params *riverdriver.TableExistsParams) (bool, error) { + ctx = informationSchemaParam(ctx) + + exists, err := dbsqlc.New().TableExists(ctx, e.dbtx, &dbsqlc.TableExistsParams{ + TableName: params.Table, + Schema: sql.NullString{String: params.Schema, Valid: params.Schema != ""}, + }) + if err != nil { + return false, interpretError(err) + } + return exists, nil +} + +func (e *Executor) TableTruncate(ctx context.Context, params *riverdriver.TableTruncateParams) error { + var maybeSchema string + if params.Schema != "" { + maybeSchema = mysqlIdentifier(params.Schema) + "." + } + + for _, table := range params.Table { + // MySQL's TRUNCATE TABLE is DDL and can't be used in transactions. + // Use DELETE FROM instead for transactional safety. + _, err := e.dbtx.ExecContext(ctx, "DELETE FROM "+maybeSchema+mysqlIdentifier(table)) + if err != nil { + return interpretError(err) + } + } + + return nil +} + +// executor preserves transaction identity when a method is invoked through an +// embedded Executor, so transaction helpers can reuse it without a savepoint. +func (e *Executor) executor() riverdriver.Executor { + if e.execTx != nil { + return e.execTx + } + return e +} + +type ExecutorTx struct { + Executor + + tx *sql.Tx +} + +func (t *ExecutorTx) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + executorSubTx := &ExecutorSubTx{ + beginOnce: &savepointutil.BeginOnlyOnce{}, + savepointNum: 0, + tx: t.tx, + } + executorSubTx.Executor = Executor{nil, templateReplaceWrapper{t.tx, &t.driver.replacer}, t.driver, executorSubTx} + return executorSubTx.Begin(ctx) +} + +func (t *ExecutorTx) Commit(ctx context.Context) error { + return t.tx.Commit() +} + +func (t *ExecutorTx) Rollback(ctx context.Context) error { + return t.tx.Rollback() +} + +type ExecutorSubTx struct { + Executor + + beginOnce *savepointutil.BeginOnlyOnce + savepointNum int + tx *sql.Tx +} + +const savepointPrefix = "river_savepoint_" + +func (t *ExecutorSubTx) Begin(ctx context.Context) (riverdriver.ExecutorTx, error) { + if err := t.beginOnce.Begin(); err != nil { + return nil, err + } + + nextSavepointNum := t.savepointNum + 1 + if err := t.Exec(ctx, fmt.Sprintf("SAVEPOINT %s%02d", savepointPrefix, nextSavepointNum)); err != nil { + return nil, err + } + + executorSubTx := &ExecutorSubTx{ + beginOnce: savepointutil.NewBeginOnlyOnce(t.beginOnce), + savepointNum: nextSavepointNum, + tx: t.tx, + } + executorSubTx.Executor = Executor{nil, templateReplaceWrapper{t.tx, &t.driver.replacer}, t.driver, executorSubTx} + + return executorSubTx, nil +} + +func (t *ExecutorSubTx) Commit(ctx context.Context) error { + defer t.beginOnce.Done() + + if t.beginOnce.IsDone() { + return errors.New("tx is closed") + } + + if err := t.Exec(ctx, fmt.Sprintf("RELEASE SAVEPOINT %s%02d", savepointPrefix, t.savepointNum)); err != nil { + // MySQL DDL statements (CREATE TABLE, ALTER TABLE, etc.) cause an + // implicit COMMIT which destroys savepoints. If the savepoint no + // longer exists, the work was already committed by the DDL. + if isSavepointDoesNotExist(err) { + return nil + } + return err + } + + return nil +} + +func (t *ExecutorSubTx) Rollback(ctx context.Context) error { + defer t.beginOnce.Done() + + if t.beginOnce.IsDone() { + return errors.New("tx is closed") + } + + if err := t.Exec(ctx, fmt.Sprintf("ROLLBACK TO SAVEPOINT %s%02d", savepointPrefix, t.savepointNum)); err != nil { + // MySQL DDL statements cause implicit COMMIT which destroys + // savepoints. If the savepoint no longer exists, there's nothing + // to roll back to. + if isSavepointDoesNotExist(err) { + return nil + } + return err + } + + return nil +} + +// isSavepointDoesNotExist checks if a MySQL error indicates that a savepoint +// does not exist (error 1305). This happens when DDL statements cause an +// implicit COMMIT that destroys active savepoints. +func isSavepointDoesNotExist(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1305 +} + +func isDuplicateEntry(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 +} + +func boolToInt64(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func interpretError(err error) error { + if errors.Is(err, sql.ErrNoRows) { + return rivertype.ErrNotFound + } + return err +} + +// mysqlIdentifier quotes an identifier with backticks for MySQL. +// MySQL uses backticks instead of double quotes for identifier quoting. +func mysqlIdentifier(ident string) string { + return "`" + strings.ReplaceAll(ident, "`", "``") + "`" +} + +type templateReplaceWrapper struct { + dbtx dbsqlc.DBTX + replacer *sqlctemplate.Replacer +} + +func (w templateReplaceWrapper) ExecContext(ctx context.Context, rawSQL string, rawArgs ...any) (sql.Result, error) { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.ExecContext(ctx, sqlStr, args...) +} + +func (w templateReplaceWrapper) PrepareContext(ctx context.Context, rawSQL string) (*sql.Stmt, error) { + sqlStr, _ := w.replacer.Run(ctx, argPlaceholder, rawSQL, nil) + return w.dbtx.PrepareContext(ctx, sqlStr) +} + +func (w templateReplaceWrapper) QueryContext(ctx context.Context, rawSQL string, rawArgs ...any) (*sql.Rows, error) { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.QueryContext(ctx, sqlStr, args...) +} + +func (w templateReplaceWrapper) QueryRowContext(ctx context.Context, rawSQL string, rawArgs ...any) *sql.Row { + sqlStr, args := w.replacer.Run(ctx, argPlaceholder, rawSQL, rawArgs) + return w.dbtx.QueryRowContext(ctx, sqlStr, args...) +} + +// jobGetAvailableResultFromInternal decodes the job rows locked by +// JobGetAvailable, retaining every row and recording decode errors by job ID +// because they've all been moved to `running`. +func jobGetAvailableResultFromInternal(jobs []*dbsqlc.RiverJob) *riverdriver.JobGetAvailableResult { + res := &riverdriver.JobGetAvailableResult{Jobs: make([]*rivertype.JobRow, len(jobs))} + for i, internal := range jobs { + job, err := jobRowFromInternalPartial(internal) + res.Jobs[i] = job + if err != nil { + if res.DecodeErrors == nil { + res.DecodeErrors = make(map[int64]error) + } + res.DecodeErrors[job.ID] = err + } + } + return res +} + +// jobRowFromInternal decodes a job row, returning an error if any of its +// fields can't be decoded. +func jobRowFromInternal(internal *dbsqlc.RiverJob) (*rivertype.JobRow, error) { + job, err := jobRowFromInternalPartial(internal) + if err != nil { + return nil, err + } + return job, nil +} + +func jobRowFromInternalPartial(internal *dbsqlc.RiverJob) (*rivertype.JobRow, error) { + attemptedAt := ptrFromNullTime(internal.AttemptedAt) + if attemptedAt != nil { + t := attemptedAt.UTC() + attemptedAt = &t + } + + var decodeErrs []error + var attemptedBy []string + if internal.AttemptedBy != nil { + if err := json.Unmarshal(internal.AttemptedBy, &attemptedBy); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `attempted_by`: %w", err)) + attemptedBy = nil + } + } + + errs := make([]rivertype.AttemptError, 0) + if internal.Errors != nil { + if err := riverdriver.UnmarshalAttemptErrors(internal.Errors, &errs); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `errors`: %w", err)) + errs = nil + } + } + + finalizedAt := ptrFromNullTime(internal.FinalizedAt) + if finalizedAt != nil { + t := finalizedAt.UTC() + finalizedAt = &t + } + + var tags []string + if err := json.Unmarshal(internal.Tags, &tags); err != nil { + decodeErrs = append(decodeErrs, fmt.Errorf("error unmarshaling `tags`: %w", err)) + tags = nil + } + + var uniqueStatesByte byte + if internal.UniqueStates.Valid { + if internal.UniqueStates.Int16 < 0 || internal.UniqueStates.Int16 > 255 { + decodeErrs = append(decodeErrs, fmt.Errorf("value out of range for byte: %d", internal.UniqueStates.Int16)) + } else { + uniqueStatesByte = byte(internal.UniqueStates.Int16) + } + } + + return &rivertype.JobRow{ + ID: internal.ID, + Attempt: max(int(internal.Attempt), 0), + AttemptedAt: attemptedAt, + AttemptedBy: attemptedBy, + CreatedAt: internal.CreatedAt.UTC(), + EncodedArgs: internal.Args, + Errors: errs, + FinalizedAt: finalizedAt, + Kind: internal.Kind, + MaxAttempts: max(int(internal.MaxAttempts), 0), + Metadata: internal.Metadata, + Priority: max(int(internal.Priority), 0), + Queue: internal.Queue, + ScheduledAt: internal.ScheduledAt.UTC(), + State: rivertype.JobState(internal.State), + Tags: tags, + UniqueKey: bytesFromNullString(internal.UniqueKey), + UniqueStates: uniquestates.UniqueBitmaskToStates(uniqueStatesByte), + }, errors.Join(decodeErrs...) +} + +// jobRowsFromInternalPartial decodes job rows with jobRowFromInternalPartial, +// ignoring decode errors so that one bad row doesn't prevent returning the +// others. +func jobRowsFromInternalPartial(jobs []*dbsqlc.RiverJob) []*rivertype.JobRow { + return sliceutil.Map(jobs, func(internal *dbsqlc.RiverJob) *rivertype.JobRow { + job, _ := jobRowFromInternalPartial(internal) + return job + }) +} + +func leaderFromInternal(internal *dbsqlc.RiverLeader) *riverdriver.Leader { + return &riverdriver.Leader{ + ElectedAt: internal.ElectedAt.UTC(), + ExpiresAt: internal.ExpiresAt.UTC(), + LeaderID: internal.LeaderID, + } +} + +func migrationFromInternal(internal *dbsqlc.RiverMigration) *riverdriver.Migration { + return &riverdriver.Migration{ + CreatedAt: internal.CreatedAt.UTC(), + Line: internal.Line, + Version: int(internal.Version), + } +} + +func schemaTemplateParam(ctx context.Context, schema string) context.Context { + if schema != "" { + schema = mysqlIdentifier(schema) + "." + } + + return sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "schema": {Value: schema}, + }, nil) +} + +func informationSchemaParam(ctx context.Context) context.Context { + return sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "information_schema": {Stable: true, Value: "INFORMATION_SCHEMA."}, + }, nil) +} + +func queueFromInternal(internal *dbsqlc.RiverQueue) *rivertype.Queue { + pausedAt := ptrFromNullTime(internal.PausedAt) + if pausedAt != nil { + t := pausedAt.UTC() + pausedAt = &t + } + return &rivertype.Queue{ + CreatedAt: internal.CreatedAt.UTC(), + Metadata: internal.Metadata, + Name: internal.Name, + PausedAt: pausedAt, + UpdatedAt: internal.UpdatedAt.UTC(), + } +} + +func nullTimeFromPtr(t *time.Time) sql.NullTime { + if t == nil { + return sql.NullTime{} + } + return sql.NullTime{Time: t.UTC(), Valid: true} +} + +func ptrFromNullTime(t sql.NullTime) *time.Time { + if !t.Valid { + return nil + } + return &t.Time +} + +func nullStringFromBytes(b []byte) sql.NullString { + if b == nil { + return sql.NullString{} + } + return sql.NullString{String: string(b), Valid: true} +} + +func bytesFromNullString(s sql.NullString) []byte { + if !s.Valid { + return nil + } + return []byte(s.String) +} + +func utcTimePtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + return new(t.UTC()) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_batch.go b/riverdriver/rivermysql/river_mysql_driver_batch.go new file mode 100644 index 000000000..c7db085e8 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_batch.go @@ -0,0 +1,299 @@ +package rivermysql + +import ( + "context" + "database/sql" + _ "embed" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" + "github.com/riverqueue/river/rivershared/uniquestates" + "github.com/riverqueue/river/rivershared/util/randutil" + "github.com/riverqueue/river/rivershared/util/sliceutil" + "github.com/riverqueue/river/rivertype" +) + +// sqlc's MySQL parser doesn't support JSON_TABLE. Keep these batch queries +// outside its input directory and use the same schema replacement and executor +// as generated queries. Single-row operations continue to use sqlc. +var ( + //go:embed internal/dbbatch/job_complete.sql + jobCompleteBatchSQL string + //go:embed internal/dbbatch/job_insert.sql + jobInsertBatchSQL string + //go:embed internal/dbbatch/job_insert_upsert.sql + jobInsertBatchUpsertSQL string + //go:embed internal/dbbatch/job_rescue.sql + jobRescueBatchSQL string + //go:embed internal/dbbatch/job_set_state.sql + jobSetStateBatchSQL string + //go:embed internal/dbbatch/job_unique_key.sql + jobUniqueKeySQL string + //go:embed internal/dbbatch/notification_insert.sql + notificationInsertBatchSQL string +) + +// A joined UPDATE only updates each matching row once. Use the sequential path +// for repeated IDs to retain the ordering of state changes and metadata merges. +func batchIDsUnique(ids []int64) bool { + seen := make(map[int64]struct{}, len(ids)) + for _, id := range ids { + if _, ok := seen[id]; ok { + return false + } + seen[id] = struct{}{} + } + return true +} + +func batchTime(t *time.Time) *string { + if t == nil { + return nil + } + return new(t.UTC().Format("2006-01-02 15:04:05.000000")) +} + +func execJSONBatch(ctx context.Context, dbtx templateReplaceWrapper, query string, rows any, args ...any) error { + encoded, err := json.Marshal(rows) + if err != nil { + return err + } + _, err = dbtx.ExecContext(ctx, query, append([]any{string(encoded)}, args...)...) + return interpretError(err) +} + +// MySQL requires a default database for joined updates containing derived +// tables, even with fully qualified table names. Retain the sequential path +// for connections without a default database; don't change pooled session state. +func isNoDatabaseSelected(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1046 +} + +func jobInsertBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobInsertFastManyParams, nonces []string) (sql.Result, error) { + // JSON documents travel as strings, then CAST back to JSON in SQL. This + // preserves JSON null (distinct from SQL NULL), large integers, and the + // database's validation of malformed JSON without decoding job arguments. + // Short field names reduce the document Go encodes and MySQL parses for + // each job. Keep these names aligned with the JSON_TABLE paths in job_insert.sql. + type row struct { + ID *int64 `json:"i"` + Args string `json:"a"` + CreatedAt *string `json:"c"` + Kind string `json:"k"` + MaxAttempts int `json:"ma"` + Metadata string `json:"m"` + Priority int `json:"p"` + Queue string `json:"q"` + ScheduledAt *string `json:"s"` + State rivertype.JobState `json:"st"` + Tags string `json:"t"` + UniqueActive bool `json:"u"` + UniqueKey string `json:"uk"` + UniqueStates byte `json:"us"` + } + rows := make([]row, len(params.Jobs)) + upsert := "" + for i, job := range params.Jobs { + tags, err := json.Marshal(job.Tags) + if err != nil { + return nil, err + } + metadata := sliceutil.FirstNonEmpty(job.Metadata, []byte("{}")) + if nonces != nil { + metadata, err = sjson.SetBytes(metadata, riverdriver.UniqueInsertMetadataKey, nonces[i]) + if err != nil { + return nil, err + } + } + rows[i] = row{ + ID: job.ID, + Args: string(job.EncodedArgs), + CreatedAt: batchTime(job.CreatedAt), + Kind: job.Kind, + MaxAttempts: job.MaxAttempts, + Metadata: string(metadata), + Priority: job.Priority, + Queue: job.Queue, + ScheduledAt: batchTime(job.ScheduledAt), + State: job.State, + Tags: string(tags), + UniqueActive: job.UniqueStates&uniquestates.UniqueStatesToBitmask([]rivertype.JobState{job.State}) != 0, + UniqueKey: hex.EncodeToString(job.UniqueKey), + UniqueStates: job.UniqueStates, + } + if len(job.UniqueKey) > 0 { + upsert = jobInsertBatchUpsertSQL + } + } + encoded, err := json.Marshal(rows) + if err != nil { + return nil, err + } + ctx = sqlctemplate.WithReplacements(ctx, map[string]sqlctemplate.Replacement{ + "on_duplicate_key": {Stable: true, Value: upsert}, + }, nil) + result, err := dbtx.ExecContext(ctx, jobInsertBatchSQL, string(encoded)) + return result, interpretError(err) +} + +func jobInsertBatchReturning(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobInsertFastManyParams, nonces []string) ([]*dbsqlc.RiverJob, error) { + // Use the first generated ID only as a lower bound, never to infer the + // remaining IDs: concurrent INSERT ... SELECT statements may interleave them. + // A per-input nonce preserves order and distinguishes newly inserted jobs. + // The caller removes these temporary nonces before committing. + prefix := randutil.Hex(16) + ":" + for i := range nonces { + nonces[i] = prefix + strconv.Itoa(i) + } + result, err := jobInsertBatch(schemaTemplateParam(ctx, params.Schema), dbtx, params, nonces) + if err != nil { + return nil, err + } + firstID, err := result.LastInsertId() + if err != nil { + return nil, err + } + jobsByNonce := make(map[string]*dbsqlc.RiverJob, len(nonces)) + if firstID != 0 { + jobs, err := dbsqlc.New().JobGetByInsertNonce(schemaTemplateParam(ctx, params.Schema), dbtx, &dbsqlc.JobGetByInsertNonceParams{ + FirstID: firstID, NoncePrefix: []byte(prefix + "%"), + }) + if err != nil { + return nil, interpretError(err) + } + for _, job := range jobs { + jobsByNonce[gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).String()] = job + } + } + + inserted := make([]*dbsqlc.RiverJob, len(nonces)) + var missingKeys []sql.NullString + for i, nonce := range nonces { + inserted[i] = jobsByNonce[nonce] + if inserted[i] == nil && len(params.Jobs[i].UniqueKey) > 0 { + missingKeys = append(missingKeys, sql.NullString{String: string(params.Jobs[i].UniqueKey), Valid: true}) + } + } + if len(missingKeys) > 0 { + lookupCtx := sqlctemplate.WithReplacements(schemaTemplateParam(ctx, params.Schema), map[string]sqlctemplate.Replacement{ + "unique_key": {Stable: true, Value: "(" + jobUniqueKeySQL + ")"}, + }, nil) + jobs, err := dbsqlc.New().JobGetByUniqueKeyManyForUpdate(lookupCtx, dbtx, missingKeys) + if err != nil { + return nil, interpretError(err) + } + jobsByKey := make(map[string]*dbsqlc.RiverJob, len(jobs)) + for _, job := range jobs { + jobsByKey[job.UniqueKey.String] = job + } + for i, job := range params.Jobs { + if inserted[i] == nil { + inserted[i] = jobsByKey[string(job.UniqueKey)] + } + } + } + for i, job := range inserted { + if job == nil { + return nil, fmt.Errorf("could not find job inserted at index %d", i) + } + } + return inserted, nil +} + +func jobInsertBatchSupported(params *riverdriver.JobInsertFastManyParams) bool { + if len(params.Jobs) < 2 { + return false + } + for _, job := range params.Jobs { + // Explicit IDs may precede the first generated ID. Preserve non-object + // metadata and caller-supplied reserved keys through the sequential path. + if job.ID != nil || (len(job.Metadata) > 0 && (!json.Valid(job.Metadata) || !gjson.ParseBytes(job.Metadata).IsObject())) || + gjson.GetBytes(job.Metadata, riverdriver.UniqueInsertMetadataKey).Exists() { + return false + } + } + return true +} + +func jobRescueBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobRescueManyParams) error { + type row struct { + ID int64 `json:"id"` + Error *string `json:"error"` + FinalizedAt *string `json:"finalized_at"` + ScheduledAt *string `json:"scheduled_at"` + State string `json:"state"` + } + rows := make([]row, len(params.ID)) + for i, id := range params.ID { + rows[i] = row{ + ID: id, + FinalizedAt: batchTime(params.FinalizedAt[i]), + ScheduledAt: batchTime(¶ms.ScheduledAt[i]), + State: params.State[i], + } + if params.Error[i] != nil { + rows[i].Error = new(string(params.Error[i])) + } + } + return execJSONBatch(ctx, dbtx, jobRescueBatchSQL, rows, params.StuckHorizon.UTC()) +} + +func jobSetStateBatch(ctx context.Context, dbtx templateReplaceWrapper, params *riverdriver.JobSetStateIfRunningManyParams) error { + completeOnly := true + for i, state := range params.State { + if state != rivertype.JobStateCompleted || params.Attempt[i] != nil || params.ErrData[i] != nil || + params.MetadataDoMerge[i] || params.ScheduledAt[i] != nil { + completeOnly = false + break + } + } + if completeOnly { + type row struct { + ID int64 `json:"id"` + FinalizedAt *string `json:"finalized_at"` + } + rows := make([]row, len(params.ID)) + for i, id := range params.ID { + rows[i] = row{ID: id, FinalizedAt: batchTime(params.FinalizedAt[i])} + } + return execJSONBatch(ctx, dbtx, jobCompleteBatchSQL, rows) + } + type row struct { + ID int64 `json:"id"` + Attempt *int `json:"attempt"` + Error *string `json:"error"` + FinalizedAt *string `json:"finalized_at"` + Metadata *string `json:"metadata"` + ScheduledAt *string `json:"scheduled_at"` + State rivertype.JobState `json:"state"` + } + rows := make([]row, len(params.ID)) + for i, id := range params.ID { + rows[i] = row{ + ID: id, + Attempt: params.Attempt[i], + FinalizedAt: batchTime(params.FinalizedAt[i]), + ScheduledAt: batchTime(params.ScheduledAt[i]), + State: params.State[i], + } + if params.ErrData[i] != nil { + rows[i].Error = new(string(params.ErrData[i])) + } + if params.MetadataDoMerge[i] { + rows[i].Metadata = new(string(params.MetadataUpdates[i])) + } + } + return execJSONBatch(ctx, dbtx, jobSetStateBatchSQL, rows, nullTimeFromPtr(params.Now)) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_batch_test.go b/riverdriver/rivermysql/river_mysql_driver_batch_test.go new file mode 100644 index 000000000..f5145f58e --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_batch_test.go @@ -0,0 +1,425 @@ +package rivermysql + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivertype" +) + +func TestJobInsertBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + exec riverdriver.Executor + params *riverdriver.JobInsertFastManyParams + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + params := &riverdriver.JobInsertFastManyParams{Schema: riverdbtest.TestSchema(ctx, t, driver, nil)} + for i := range 2 { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + ID: new(int64(9007199254740993 + i)), EncodedArgs: []byte(`null`), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`null`), Priority: 1, + Queue: "default", State: rivertype.JobStateAvailable, + }) + } + return &testBundle{exec: driver.GetExecutor(), params: params} + } + + t.Run("JSONNullsAndIDs", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + bundle.params.Jobs[1].EncodedArgs = []byte(`{"id":9007199254740993}`) + bundle.params.Jobs[1].Metadata = []byte(`{"id":9007199254740993}`) + count, err := bundle.exec.JobInsertFastManyNoReturning(ctx, bundle.params) + require.NoError(t, err) + require.Equal(t, 2, count) + for i, params := range bundle.params.Jobs { + job, err := bundle.exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: *params.ID, Schema: bundle.params.Schema}) + require.NoError(t, err) + require.Equal(t, *params.ID, job.ID) + require.JSONEq(t, string(params.EncodedArgs), string(job.EncodedArgs)) + require.JSONEq(t, string(params.Metadata), string(job.Metadata)) + if i == 1 { + require.Contains(t, string(job.EncodedArgs), "9007199254740993") + require.Contains(t, string(job.Metadata), "9007199254740993") + } + require.Nil(t, job.Tags) + } + }) + + t.Run("RejectInvalidBatch", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + bundle.params.Jobs[1].EncodedArgs = []byte(`{`) + _, err = tx.JobInsertFastManyNoReturning(ctx, bundle.params) + require.Error(t, err) + count, err := tx.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) + require.Zero(t, count) + }) +} + +func TestJobInsertBatchReturning(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + driver *Driver + params *riverdriver.JobInsertFastManyParams + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + params := &riverdriver.JobInsertFastManyParams{Schema: riverdbtest.TestSchema(ctx, t, driver, nil)} + for i := range byte(2) { + params.Jobs = append(params.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"index":%d}`, i)), + Kind: "batch", MaxAttempts: 25, Metadata: []byte(`{"id":9007199254740993}`), + Priority: 1, Queue: "default", State: rivertype.JobStateAvailable, + UniqueKey: []byte{0xff, 0, i}, UniqueStates: 0xff, + }) + } + return &testBundle{driver: driver, params: params} + } + + t.Run("AutoIncrementGaps", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = tx.ExecContext(context.Background(), "SET SESSION auto_increment_increment = DEFAULT") + _ = tx.Rollback() + }) + _, err = tx.ExecContext(ctx, "SET SESSION auto_increment_increment = 7") + require.NoError(t, err) + results, err := bundle.driver.UnwrapExecutor(tx).JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + require.Equal(t, int64(7), results[1].Job.ID-results[0].Job.ID) + for i, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(bundle.params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + require.JSONEq(t, string(bundle.params.Jobs[i].Metadata), string(result.Job.Metadata)) + stored, err := bundle.driver.UnwrapExecutor(tx).JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: result.Job.ID, Schema: bundle.params.Schema, + }) + require.NoError(t, err) + require.JSONEq(t, string(bundle.params.Jobs[i].Metadata), string(stored.Metadata)) + } + }) + + t.Run("ConcurrentBatches", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for worker := range 4 { + t.Run(fmt.Sprintf("Worker%d", worker), func(t *testing.T) { + t.Parallel() + + for iteration := range 10 { + params := &riverdriver.JobInsertFastManyParams{Schema: bundle.params.Schema} + for i := range 20 { + job := *bundle.params.Jobs[0] + job.EncodedArgs = fmt.Appendf(nil, `{"worker":%d,"iteration":%d,"index":%d}`, worker, iteration, i) + job.UniqueKey = nil + if i%2 == 0 { + job.UniqueKey = fmt.Appendf(nil, "%d/%d/%d", worker, iteration, i) + } + params.Jobs = append(params.Jobs, &job) + } + results, err := bundle.driver.GetExecutor().JobInsertFastMany(t.Context(), params) + require.NoError(t, err) + require.Len(t, results, len(params.Jobs)) + for i, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, string(params.Jobs[i].EncodedArgs), string(result.Job.EncodedArgs)) + } + } + }) + } + }) + + t.Run("CurrentRead", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(ctx) }) + _, err = tx.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) // Establish a snapshot before the duplicate exists. + seed, err := bundle.driver.GetExecutor().JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: bundle.params.Jobs[:1], Schema: bundle.params.Schema, + }) + require.NoError(t, err) + + results, err := tx.JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + require.True(t, results[0].UniqueSkippedAsDuplicate) + require.Equal(t, seed[0].Job.ID, results[0].Job.ID) + require.False(t, results[1].UniqueSkippedAsDuplicate) + }) + + t.Run("NonObjectAndReservedMetadata", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for _, metadata := range []string{`null`, `[1,2]`, `{"river:unique_nonce":"caller"}`} { + for _, job := range bundle.params.Jobs { + job.Metadata = []byte(metadata) + job.UniqueKey = nil + } + results, err := bundle.driver.GetExecutor().JobInsertFastMany(t.Context(), bundle.params) + require.NoError(t, err) + require.Len(t, results, 2) + for _, result := range results { + require.False(t, result.UniqueSkippedAsDuplicate) + require.JSONEq(t, metadata, string(result.Job.Metadata)) + } + } + }) + + t.Run("ReadDoesNotLockOtherInserts", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + for _, job := range bundle.params.Jobs { + job.UniqueKey = nil + } + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + dbtx := templateReplaceWrapper{dbtx: tx, replacer: &bundle.driver.replacer} + result, err := jobInsertBatch(schemaTemplateParam(ctx, bundle.params.Schema), dbtx, bundle.params, []string{"test:0", "test:1"}) + require.NoError(t, err) + firstID, err := result.LastInsertId() + require.NoError(t, err) + + otherTx, err := bundle.driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = otherTx.Rollback(ctx) }) + _, err = otherTx.JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + + // The primary-key range includes another session's uncommitted jobs. + // Locking the whole range here could deadlock concurrent insertions. + readCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + rows, err := dbsqlc.New().JobGetByInsertNonce(schemaTemplateParam(readCtx, bundle.params.Schema), dbtx, + &dbsqlc.JobGetByInsertNonceParams{FirstID: firstID, NoncePrefix: []byte("test:%")}) + require.NoError(t, err) + require.Len(t, rows, 2) + }) + + t.Run("UniqueLookupUsesIndex", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + _, err := bundle.driver.GetExecutor().JobInsertFastMany(ctx, bundle.params) + require.NoError(t, err) + var plan string + err = bundle.driver.dbPool.QueryRowContext(ctx, + "EXPLAIN FORMAT=JSON SELECT id FROM "+mysqlIdentifier(bundle.params.Schema)+".river_job WHERE ("+jobUniqueKeySQL+") IN (?, ?)", + bundle.params.Jobs[0].UniqueKey, bundle.params.Jobs[1].UniqueKey).Scan(&plan) + require.NoError(t, err) + // MySQL 9 uses a new JSON plan format; both formats identify the + // actual index separately from the list of possible indexes. + require.Regexp(t, `"(?:key|index_name)":\s*"river_job_unique_idx"`, plan) + }) + + t.Run("UnrelatedUniqueConflictIsAtomic", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + exec := bundle.driver.GetExecutor() + require.NoError(t, exec.Exec(ctx, "CREATE UNIQUE INDEX kind_unique ON "+mysqlIdentifier(bundle.params.Schema)+".river_job (kind)")) + t.Cleanup(func() { + require.NoError(t, exec.Exec(context.Background(), "DROP INDEX kind_unique ON "+mysqlIdentifier(bundle.params.Schema)+".river_job")) + }) + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: bundle.params.Jobs[:1], Schema: bundle.params.Schema}) + require.NoError(t, err) + bundle.params.Jobs[0].Kind = "new_kind" + bundle.params.Jobs[0].UniqueKey = []byte("new_key") + + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = tx.ExecContext(context.Background(), "SET SESSION sql_mode = DEFAULT") + _ = tx.Rollback() + }) + // Conflict rejection must not depend on strict SQL mode. + _, err = tx.ExecContext(ctx, "SET SESSION sql_mode = ''") + require.NoError(t, err) + txExec := bundle.driver.UnwrapExecutor(tx) + _, err = txExec.JobInsertFastMany(ctx, bundle.params) + require.ErrorContains(t, err, "Invalid JSON text") + count, err := txExec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{Schema: bundle.params.Schema, State: rivertype.JobStateAvailable}) + require.NoError(t, err) + require.Equal(t, 1, count) + }) +} + +func TestJobSetStateBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type testBundle struct { + driver *Driver + jobs []*rivertype.JobRow + params *riverdriver.JobSetStateIfRunningManyParams + schema string + } + setup := func(t *testing.T) *testBundle { + t.Helper() + + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + schema := riverdbtest.TestSchema(ctx, t, driver, nil) + params := &riverdriver.JobSetStateIfRunningManyParams{Schema: schema} + jobs := make([]*rivertype.JobRow, 0, 2) + for range 2 { + job, err := driver.GetExecutor().JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + Attempt: 3, EncodedArgs: []byte(`{}`), Kind: "batch", MaxAttempts: 25, + Priority: 1, Queue: "default", Schema: schema, State: rivertype.JobStateRunning, Tags: []string{}, + }) + require.NoError(t, err) + jobs = append(jobs, job) + params.ID = append(params.ID, job.ID) + params.Attempt = append(params.Attempt, nil) + params.ErrData = append(params.ErrData, nil) + params.FinalizedAt = append(params.FinalizedAt, nil) + params.MetadataDoMerge = append(params.MetadataDoMerge, true) + params.MetadataUpdates = append(params.MetadataUpdates, []byte(`{"result":"merged"}`)) + params.ScheduledAt = append(params.ScheduledAt, new(time.Now().Add(time.Hour))) + params.State = append(params.State, rivertype.JobStateRetryable) + } + return &testBundle{driver: driver, jobs: jobs, params: params, schema: schema} + } + + t.Run("CompletionUsesCurrentRead", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + // Exercise the joined query even when the DSN has no default database. + _, err = tx.ExecContext(ctx, "USE "+mysqlIdentifier(bundle.schema)) + require.NoError(t, err) + exec := bundle.driver.UnwrapExecutor(tx) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: bundle.jobs[0].ID, Schema: bundle.schema}) + require.NoError(t, err) // Establish a REPEATABLE READ snapshot. + + now := time.Now().UTC().Truncate(time.Microsecond) + previousFinalizedAt := now.Add(-time.Minute) + _, err = bundle.driver.GetExecutor().JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: bundle.jobs[0].ID, CancelAttemptedAt: now, Now: &now, Schema: bundle.schema, + }) + require.NoError(t, err) + require.NoError(t, bundle.driver.GetExecutor().Exec(ctx, + "UPDATE "+mysqlIdentifier(bundle.schema)+".river_job SET state = 'completed', finalized_at = ? WHERE id = ?", + previousFinalizedAt, bundle.jobs[1].ID)) + + for i := range bundle.params.ID { + bundle.params.FinalizedAt[i] = &now + bundle.params.MetadataDoMerge[i] = false + bundle.params.MetadataUpdates[i] = nil + bundle.params.ScheduledAt[i] = nil + bundle.params.State[i] = rivertype.JobStateCompleted + } + rows, err := exec.JobSetStateIfRunningMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, rows, 2) + require.Equal(t, rivertype.JobStateCompleted, rows[0].State) + require.Equal(t, now, *rows[0].FinalizedAt) + require.Contains(t, string(rows[0].Metadata), "cancel_attempted_at") + require.Equal(t, rivertype.JobStateCompleted, rows[1].State) + require.Equal(t, previousFinalizedAt, *rows[1].FinalizedAt) + require.JSONEq(t, `{}`, string(rows[1].Metadata)) + }) + + t.Run("CurrentRead", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + ctx := t.Context() + tx, err := bundle.driver.dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + // Ensure the joined batch query is used even if TEST_MYSQL_URL has no database. + _, err = tx.ExecContext(ctx, "USE "+mysqlIdentifier(bundle.schema)) + require.NoError(t, err) + exec := bundle.driver.UnwrapExecutor(tx) + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ID: bundle.jobs[0].ID, Schema: bundle.schema}) + require.NoError(t, err) // Establish a REPEATABLE READ snapshot before the changes below. + + now := time.Now().UTC().Truncate(time.Microsecond) + _, err = bundle.driver.GetExecutor().JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: bundle.jobs[0].ID, CancelAttemptedAt: now, Now: &now, Schema: bundle.schema, + }) + require.NoError(t, err) + require.NoError(t, bundle.driver.GetExecutor().Exec(ctx, + "UPDATE "+mysqlIdentifier(bundle.schema)+".river_job SET state = 'completed', finalized_at = ? WHERE id = ?", now, bundle.jobs[1].ID)) + + rows, err := exec.JobSetStateIfRunningMany(ctx, bundle.params) + require.NoError(t, err) + require.Len(t, rows, 2) + require.Equal(t, rivertype.JobStateCancelled, rows[0].State) + require.Equal(t, bundle.jobs[0].ScheduledAt, rows[0].ScheduledAt) + require.Equal(t, rivertype.JobStateCompleted, rows[1].State) + require.Equal(t, now, *rows[1].FinalizedAt) + require.JSONEq(t, `{"result":"merged"}`, string(rows[1].Metadata)) + }) + + t.Run("RepeatedIDs", func(t *testing.T) { + t.Parallel() + + bundle := setup(t) + bundle.params.ID[1] = bundle.params.ID[0] + bundle.params.State[1] = rivertype.JobStateScheduled + bundle.params.MetadataUpdates[0] = []byte(`{"first":true}`) + bundle.params.MetadataUpdates[1] = []byte(`{"second":true}`) + rows, err := bundle.driver.GetExecutor().JobSetStateIfRunningMany(t.Context(), bundle.params) + require.NoError(t, err) + require.Len(t, rows, 2) + for _, row := range rows { + require.Equal(t, rivertype.JobStateRetryable, row.State) + require.JSONEq(t, `{"first":true,"second":true}`, string(row.Metadata)) + } + }) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go b/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go new file mode 100644 index 000000000..d77937d46 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_benchmark_test.go @@ -0,0 +1,236 @@ +package rivermysql + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivertype" +) + +// BenchmarkJSONTableBatch includes a transaction and rollback in each iteration, +// so updates always act on running jobs and inserts don't grow the tables. +func BenchmarkJSONTableBatch(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + now := time.Now().UTC().Truncate(time.Microsecond) + attemptedAt := now.Add(-time.Hour) + + for _, operation := range []string{"Complete", "Insert", "Notify", "Rescue", "SetState"} { + for _, size := range []int{1, 10, 100, 1000} { + b.Run(fmt.Sprintf("%s/%d", operation, size), func(b *testing.B) { + insertParams := &riverdriver.JobInsertFastManyParams{Schema: schema} + notifyParams := &riverdriver.NotifyManyParams{Schema: schema, Topic: "river_insert"} + rescueParams := &riverdriver.JobRescueManyParams{Schema: schema, StuckHorizon: now} + stateParams := &riverdriver.JobSetStateIfRunningManyParams{Schema: schema} + for range size { + insertParams.Jobs = append(insertParams.Jobs, &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"message":"benchmark","account_id":123456789}`), + Kind: "benchmark", + MaxAttempts: 25, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"benchmark"}, + }) + notifyParams.Payload = append(notifyParams.Payload, `{"queue":"default"}`) + if operation != "Complete" && operation != "Rescue" && operation != "SetState" { + continue + } + jobs, err := exec.JobInsertFullMany(ctx, &riverdriver.JobInsertFullManyParams{ + Jobs: []*riverdriver.JobInsertFullParams{{ + Attempt: 1, AttemptedAt: &attemptedAt, EncodedArgs: []byte(`{}`), + Kind: "benchmark", MaxAttempts: 25, Priority: 1, Queue: "default", + State: rivertype.JobStateRunning, Tags: []string{}, + }}, + Schema: schema, + }) + require.NoError(b, err) + id := jobs[0].ID + rescueParams.ID = append(rescueParams.ID, id) + rescueParams.Error = append(rescueParams.Error, []byte(`{"attempt":1,"error":"stuck job"}`)) + rescueParams.FinalizedAt = append(rescueParams.FinalizedAt, nil) + rescueParams.ScheduledAt = append(rescueParams.ScheduledAt, now) + rescueParams.State = append(rescueParams.State, string(rivertype.JobStateAvailable)) + stateParams.ID = append(stateParams.ID, id) + stateParams.Attempt = append(stateParams.Attempt, nil) + stateParams.ErrData = append(stateParams.ErrData, nil) + stateParams.FinalizedAt = append(stateParams.FinalizedAt, &now) + stateParams.MetadataDoMerge = append(stateParams.MetadataDoMerge, operation != "Complete") + stateParams.MetadataUpdates = append(stateParams.MetadataUpdates, []byte(`{"result":"ok"}`)) + stateParams.ScheduledAt = append(stateParams.ScheduledAt, nil) + stateParams.State = append(stateParams.State, rivertype.JobStateCompleted) + } + + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + switch operation { + case "Insert": + count, err := tx.JobInsertFastManyNoReturning(ctx, insertParams) + require.NoError(b, err) + require.Equal(b, size, count) + case "Notify": + require.NoError(b, tx.NotifyMany(ctx, notifyParams)) + case "Rescue": + _, err := tx.JobRescueMany(ctx, rescueParams) + require.NoError(b, err) + case "Complete", "SetState": + jobs, err := tx.JobSetStateIfRunningMany(ctx, stateParams) + require.NoError(b, err) + require.Len(b, jobs, size) + } + require.NoError(b, tx.Rollback(ctx)) + } + }) + } + } +} + +func BenchmarkJobGetAvailableKnownKinds(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + kinds := make([]string, 100) + for i := range kinds { + kinds[i] = fmt.Sprintf("kind_%03d", i) + } + jobs := make([]*riverdriver.JobInsertFastParams, 10_000) + for i := range jobs { + jobs[i] = &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{}`), + Kind: kinds[i%len(kinds)], + MaxAttempts: 1, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + } + } + _, err := exec.JobInsertFastManyNoReturning(ctx, &riverdriver.JobInsertFastManyParams{Jobs: jobs, Schema: schema}) + require.NoError(b, err) + + for _, testCase := range []struct { + kinds []string + name string + }{ + {kinds: kinds, name: "AllKnown100Kinds"}, + {kinds: nil, name: "Disabled"}, + {kinds: kinds[:1], name: "OnePercentKnown"}, + } { + b.Run(testCase.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + result, err := tx.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "benchmark", + Kind: testCase.kinds, + MaxAttemptedBy: 4, + MaxToLock: 100, + Queue: "default", + Schema: schema, + }) + require.NoError(b, err) + require.Len(b, result.Jobs, 100) + require.NoError(b, tx.Rollback(ctx)) + } + }) + } +} + +// BenchmarkJobInsertManyReturning measures the path used by standard InsertMany, +// including fetching returned jobs and classifying uniqueness conflicts. +func BenchmarkJobInsertManyReturning(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, b)) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + exec := driver.GetExecutor() + for _, mode := range []string{"MixedDuplicates", "Plain", "Unique"} { + for _, size := range []int{1, 10, 100, 1000} { + b.Run(fmt.Sprintf("%s/%d", mode, size), func(b *testing.B) { + params := &riverdriver.JobInsertFastManyParams{Schema: schema} + for i := range size { + job := &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(fmt.Sprintf(`{"message":"benchmark","index":%d}`, i)), + Kind: "benchmark", + MaxAttempts: 25, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"benchmark"}, + } + if mode != "Plain" { + job.UniqueKey = fmt.Appendf(nil, "%s/%d/%d", mode, size, i) + job.UniqueStates = 0xff + } + params.Jobs = append(params.Jobs, job) + if mode == "MixedDuplicates" && i%2 == 0 { + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: []*riverdriver.JobInsertFastParams{job}, Schema: schema}) + require.NoError(b, err) + } + } + + b.ReportAllocs() + for b.Loop() { + tx, err := exec.Begin(ctx) + require.NoError(b, err) + jobs, err := tx.JobInsertFastMany(ctx, params) + require.NoError(b, err) + require.Len(b, jobs, size) + require.NoError(b, tx.Rollback(ctx)) + } + }) + } + } +} + +func BenchmarkListenerNotificationDrain(b *testing.B) { + riversharedtest.SkipIfMySQLNotEnabled(b) + + ctx := b.Context() + pool := riversharedtest.DBPoolMySQL(ctx, b) + driver := New(pool) + schema := riverdbtest.TestSchema(ctx, b, driver, nil) + payloads := make([]string, 256) + for i := range payloads { + payloads[i] = strconv.Itoa(i) + } + require.NoError(b, driver.GetExecutor().NotifyMany(ctx, &riverdriver.NotifyManyParams{ + Payload: payloads, + Schema: schema, + Topic: "benchmark", + })) + + b.ReportAllocs() + for b.Loop() { + listener := &Listener{ + dbPool: pool, + isConnected: true, + replacer: &driver.replacer, + schema: schema, + topics: map[string]int64{"benchmark": 0}, + } + for _, payload := range payloads { + notification, err := listener.WaitForNotification(ctx) + require.NoError(b, err) + require.Equal(b, payload, notification.Payload) + } + } +} diff --git a/riverdriver/rivermysql/river_mysql_driver_json.go b/riverdriver/rivermysql/river_mysql_driver_json.go new file mode 100644 index 000000000..93647459b --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_json.go @@ -0,0 +1,157 @@ +package rivermysql + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + "text/template" + + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" +) + +// sqlc's MySQL parser does not support JSON_TABLE. Keep these predicate +// templates outside its input directory, alongside the other driver SQL. +// +//go:embed internal/dbfragment/json_contains.sql +var jsonContainsSQL string + +// Parsed once and used without mutation for all metadata filters. +var jsonContainsTemplates = template.Must(template.New("json_contains").Parse(jsonContainsSQL)) //nolint:gochecknoglobals + +// Merge under row locks before updating. JSON_MERGE_PATCH recursively merges +// objects and removes JSON nulls, while River replaces each supplied member. +// Read all metadata in one query to retain batching and preserve repeated IDs. +func jobMetadataMergeMany(ctx context.Context, dbtx templateReplaceWrapper, ids []int64, doMerge []bool, updates [][]byte) ([][]byte, error) { + var mergeIDs []int64 + for i, id := range ids { + if doMerge[i] { + mergeIDs = append(mergeIDs, id) + } + } + if len(mergeIDs) == 0 { + return updates, nil + } + rows, err := dbsqlc.New().JobGetMetadataForUpdate(ctx, dbtx, mergeIDs) + if err != nil { + return nil, err + } + metadata := make(map[int64]json.RawMessage, len(rows)) + for _, row := range rows { + metadata[row.ID] = json.RawMessage(row.Metadata) + } + merged := slices.Clone(updates) + for i, id := range ids { + value, exists := metadata[id] + if !doMerge[i] || !exists { + continue + } + merged[i], err = jsonMerge(value, updates[i]) + if err != nil { + return nil, fmt.Errorf("merging job %d metadata: %w", id, err) + } + metadata[id] = merged[i] + } + return merged, nil +} + +// JSON_CONTAINS flattens nested arrays. Traverse each level explicitly to +// preserve Postgres containment boundaries, including whole object elements. +// Scalar subqueries avoid MySQL 8 semijoin rewrites losing nested correlations. +func jsonContains(source string, fragment json.RawMessage, bindFunc func(any) string, depth int) (string, error) { + fragment = bytes.TrimSpace(fragment) + values := map[string]string{"source": source} + query := "scalar" + switch fragment[0] { + case '{', '[': + alias := fmt.Sprintf("metadata_%d", depth) + var children []string + childFunc := func(query, param string, child json.RawMessage) error { + predicate, err := jsonContains(alias+".value", child, bindFunc, depth+1) + if err != nil { + return err + } + var sql strings.Builder + err = jsonContainsTemplates.ExecuteTemplate(&sql, query, map[string]string{ + "alias": alias, "param": param, "predicate": predicate, "source": source, + }) + children = append(children, sql.String()) + return err + } + if fragment[0] == '{' { + values["kind"] = "OBJECT" + var object map[string]json.RawMessage + if err := json.Unmarshal(fragment, &object); err != nil { + return "", err + } + for _, key := range slices.Sorted(maps.Keys(object)) { + encodedKey, err := json.Marshal(key) + if err != nil { + return "", err + } + if err := childFunc("member", bindFunc("$."+string(encodedKey)), object[key]); err != nil { + return "", err + } + } + } else { + values["kind"] = "ARRAY" + var array []json.RawMessage + if err := json.Unmarshal(fragment, &array); err != nil { + return "", err + } + for _, child := range array { + if err := childFunc("element", "", child); err != nil { + return "", err + } + } + } + values["children"] = "true" + if len(children) > 0 { + values["children"] = strings.Join(children, " AND ") + } + query = "container" + case 'n': + query = "null" + default: + values["param"] = bindFunc(string(fragment)) + } + var sql strings.Builder + err := jsonContainsTemplates.ExecuteTemplate(&sql, query, values) + return sql.String(), err +} + +func jsonMerge(left, right json.RawMessage) ([]byte, error) { + left, right = bytes.TrimSpace(left), bytes.TrimSpace(right) + if !json.Valid(left) || !json.Valid(right) { + return nil, errors.New("invalid JSON metadata") + } + if left[0] == '{' && right[0] == '{' { + var original, updates map[string]json.RawMessage + if err := json.Unmarshal(left, &original); err != nil { + return nil, err + } + if err := json.Unmarshal(right, &updates); err != nil { + return nil, err + } + maps.Copy(original, updates) + return json.Marshal(original) + } + values := make([]json.RawMessage, 0) + for _, value := range []json.RawMessage{left, right} { + if value[0] == '[' { + var elements []json.RawMessage + if err := json.Unmarshal(value, &elements); err != nil { + return nil, err + } + values = append(values, elements...) + } else { + values = append(values, value) + } + } + return json.Marshal(values) +} diff --git a/riverdriver/rivermysql/river_mysql_driver_test.go b/riverdriver/rivermysql/river_mysql_driver_test.go new file mode 100644 index 000000000..6fc80ad09 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_driver_test.go @@ -0,0 +1,1123 @@ +package rivermysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + + "github.com/riverqueue/river/riverdbtest" + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/rivershared/riversharedtest" + "github.com/riverqueue/river/rivershared/testsignal" + "github.com/riverqueue/river/rivertype" +) + +// Verify interface compliance. +var _ riverdriver.Driver[*sql.Tx] = New(nil) + +func TestIsDuplicateEntry(t *testing.T) { + t.Parallel() + + require.True(t, isDuplicateEntry(&mysql.MySQLError{Number: 1062})) + require.False(t, isDuplicateEntry(&mysql.MySQLError{Number: 1305})) + require.False(t, isDuplicateEntry(errors.New("unrelated error containing 1062"))) +} + +func TestIsSavepointDoesNotExist(t *testing.T) { + t.Parallel() + + require.True(t, isSavepointDoesNotExist(&mysql.MySQLError{Number: 1305})) + require.False(t, isSavepointDoesNotExist(&mysql.MySQLError{Number: 1062})) + require.False(t, isSavepointDoesNotExist(errors.New("unrelated error containing 1305"))) +} + +func TestInterpretError(t *testing.T) { + t.Parallel() + + require.EqualError(t, interpretError(errors.New("an error")), "an error") + require.ErrorIs(t, interpretError(sql.ErrNoRows), rivertype.ErrNotFound) + require.NoError(t, interpretError(nil)) +} + +func TestSchemaTemplateParam(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("NoSchema", func(t *testing.T) { + t.Parallel() + ctx := schemaTemplateParam(ctx, "") + // Just verify it doesn't panic + _ = ctx + }) + + t.Run("WithSchema", func(t *testing.T) { + t.Parallel() + ctx := schemaTemplateParam(ctx, "custom_schema") + _ = ctx + }) +} + +func TestDriverProperties(t *testing.T) { + t.Parallel() + + driver := New(nil) + require.Equal(t, "?", driver.ArgPlaceholder()) + require.Equal(t, riverdriver.DatabaseNameMySQL, driver.DatabaseName()) + require.True(t, driver.SupportsListener()) + require.True(t, driver.SupportsListenNotify()) + require.Equal(t, time.Microsecond, driver.TimePrecision()) + require.False(t, driver.PoolIsSet()) +} + +func TestJobInsertFastManyUniqueConflictWithinBatch(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + newJobParams := func() *riverdriver.JobInsertFastParams { + return &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"encoded": "args"}`), + Kind: "test_kind", + MaxAttempts: 25, + Metadata: []byte(`{"meta": "data"}`), + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"tag"}, + UniqueKey: []byte("unique-key-fast-conflict-within-batch"), + UniqueStates: 0xff, + } + } + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(), newJobParams()}, + Schema: schema, + }) + require.ErrorContains(t, err, "unique key appears more than once in batch") + require.Empty(t, results) +} + +func TestJobInsertFastManyUnrelatedUniqueConstraint(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + indexName = "river_job_test_kind_unique" + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_job") + ) + + require.NoError(t, exec.Exec(ctx, "CREATE UNIQUE INDEX "+mysqlIdentifier(indexName)+" ON "+table+" (kind)")) + t.Cleanup(func() { + require.NoError(t, exec.Exec(context.Background(), "DROP INDEX "+mysqlIdentifier(indexName)+" ON "+table)) + }) + + newJobParams := func(uniqueStates byte) *riverdriver.JobInsertFastParams { + return &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{"encoded": "args"}`), + Kind: "test_kind", + MaxAttempts: 25, + Metadata: []byte(`{"meta": "data"}`), + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{"tag"}, + UniqueKey: []byte("same-unique-key"), + UniqueStates: uniqueStates, + } + } + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(0xff)}, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 1) + + _, err = exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + // The requested available job's River uniqueness constraint is inactive, + // so matching the existing key must not hide the unrelated kind conflict. + Jobs: []*riverdriver.JobInsertFastParams{newJobParams(1 << 2)}, + Schema: schema, + }) + require.EqualError(t, err, "job insert at index 0 conflicted on a non-River unique constraint") + + jobs, err := exec.JobGetByKindMany(ctx, &riverdriver.JobGetByKindManyParams{Kind: []string{"test_kind"}, Schema: schema}) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, []byte("same-unique-key"), jobs[0].UniqueKey) +} + +func TestJobInsertAndGet(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{"test": true}`), + Kind: "test_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{"tag1"}, + }) + require.NoError(t, err) + require.NotNil(t, job) + require.Positive(t, job.ID) + require.Equal(t, "test_job", job.Kind) + require.Equal(t, rivertype.JobStateAvailable, job.State) + require.Equal(t, "default", job.Queue) + require.Equal(t, 1, job.Priority) + require.Equal(t, 3, job.MaxAttempts) + require.JSONEq(t, `{"test": true}`, string(job.EncodedArgs)) + require.Equal(t, []string{"tag1"}, job.Tags) + + // Get the job by ID + fetched, err := exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: job.ID, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, job.ID, fetched.ID) + require.Equal(t, job.Kind, fetched.Kind) +} + +func TestJobGetAvailable(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert some available jobs + for i := range 3 { + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: fmt.Sprintf("job_%d", i), + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + } + + // Get available jobs (needs transaction for FOR UPDATE) + txExec, err := exec.Begin(ctx) + require.NoError(t, err) + defer txExec.Rollback(ctx) + + jobs, err := txExec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "test-client", + MaxAttemptedBy: 4, + MaxToLock: 2, + Queue: "default", + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, jobs.Jobs, 2) + + for _, job := range jobs.Jobs { + require.Equal(t, rivertype.JobStateRunning, job.State) + } + + require.NoError(t, txExec.Commit(ctx)) +} + +func TestJobGetAvailableConcurrentClaimsAreUnique(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + const ( + numJobs = 50 + numWorkers = 10 + ) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + jobs := make([]*riverdriver.JobInsertFastParams, numJobs) + for i := range jobs { + jobs[i] = &riverdriver.JobInsertFastParams{ + EncodedArgs: []byte(`{}`), + Kind: "concurrent_claim", + MaxAttempts: 3, + Priority: 1, + Queue: "concurrent_claims", + State: rivertype.JobStateAvailable, + Tags: []string{}, + } + } + _, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{Jobs: jobs, Schema: schema}) + require.NoError(t, err) + + var ( + claimedIDs []int64 + errs []error + mu sync.Mutex + ready sync.WaitGroup + start = make(chan struct{}) + workers sync.WaitGroup + ) + ready.Add(numWorkers) + workers.Add(numWorkers) + for range numWorkers { + go func() { + defer workers.Done() + ready.Done() + <-start + + claimed, err := exec.JobGetAvailable(ctx, &riverdriver.JobGetAvailableParams{ + ClientID: "concurrent-client", + MaxAttemptedBy: 4, + MaxToLock: numJobs / numWorkers, + Queue: "concurrent_claims", + Schema: schema, + }) + + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, err) + return + } + for _, job := range claimed.Jobs { + claimedIDs = append(claimedIDs, job.ID) + } + }() + } + + ready.Wait() + close(start) + workers.Wait() + + require.NoError(t, errors.Join(errs...)) + require.Len(t, claimedIDs, numJobs) + uniqueClaimedIDs := make(map[int64]struct{}, len(claimedIDs)) + for _, id := range claimedIDs { + uniqueClaimedIDs[id] = struct{}{} + } + require.Len(t, uniqueClaimedIDs, numJobs) +} + +func TestJobDeleteWaitsForConcurrentStateChange(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "delete_concurrent_state_change", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + tx, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + + _, err = tx.ExecContext(ctx, "UPDATE "+mysqlIdentifier(schema)+".river_job SET state = 'running' WHERE id = ?", job.ID) //nolint:gosec + require.NoError(t, err) + + type deleteResult struct { + err error + job *rivertype.JobRow + } + deleteFinished := testsignal.TestSignal[deleteResult]{} + deleteFinished.Init(t) + deleteStarted := testsignal.TestSignal[struct{}]{} + deleteStarted.Init(t) + + go func() { + deleteStarted.Signal(struct{}{}) + job, err := exec.JobDelete(ctx, &riverdriver.JobDeleteParams{ID: job.ID, Schema: schema}) + deleteFinished.Signal(deleteResult{err: err, job: job}) + }() + + deleteStarted.WaitOrTimeout() + select { + case <-deleteFinished.WaitC(): + require.FailNow(t, "JobDelete returned while the concurrent update was uncommitted") + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, tx.Commit()) + + result := deleteFinished.WaitOrTimeout() + require.ErrorIs(t, result.err, rivertype.ErrJobRunning) + require.Nil(t, result.job) +} + +func TestJobCancel(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "test_cancel", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + // Cancel the job + cancelled, err := exec.JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: job.ID, + CancelAttemptedAt: time.Now().UTC(), + ControlTopic: "test_topic", + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateCancelled, cancelled.State) + require.NotNil(t, cancelled.FinalizedAt) +} + +func TestJobDelete(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a completed job + now := time.Now().UTC().Truncate(time.Microsecond) + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + FinalizedAt: &now, + Kind: "test_delete", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateCompleted, + Tags: []string{}, + }) + require.NoError(t, err) + + // Delete the job + deleted, err := exec.JobDelete(ctx, &riverdriver.JobDeleteParams{ + ID: job.ID, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, job.ID, deleted.ID) + + // Verify it's gone + _, err = exec.JobGetByID(ctx, &riverdriver.JobGetByIDParams{ + ID: job.ID, + Schema: schema, + }) + require.ErrorIs(t, err, rivertype.ErrNotFound) +} + +func TestJobCountByState(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert jobs in different states + for range 3 { + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "test_count", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + } + + count, err := exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 3, count) +} + +func TestLeaderElection(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Attempt to elect a leader + leader, err := exec.LeaderAttemptElect(ctx, &riverdriver.LeaderElectParams{ + LeaderID: "test-leader", + Schema: schema, + TTL: 30 * time.Second, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", leader.LeaderID) + + // Get the elected leader + fetched, err := exec.LeaderGetElectedLeader(ctx, &riverdriver.LeaderGetElectedLeaderParams{ + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", fetched.LeaderID) + + // Re-elect should succeed + reelected, err := exec.LeaderAttemptReelect(ctx, &riverdriver.LeaderReelectParams{ + ElectedAt: leader.ElectedAt, + LeaderID: "test-leader", + Schema: schema, + TTL: 30 * time.Second, + }) + require.NoError(t, err) + require.Equal(t, "test-leader", reelected.LeaderID) + + // Resign + resigned, err := exec.LeaderResign(ctx, &riverdriver.LeaderResignParams{ + ElectedAt: leader.ElectedAt, + LeaderID: "test-leader", + LeadershipTopic: "leadership", + Schema: schema, + }) + require.NoError(t, err) + require.True(t, resigned) +} + +func TestListenerConnectDoesNotSkipLowerUncommittedID(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: schema}).(*Listener) //nolint:forcetypeassert + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_notification") + ) + insertNotificationSQL := "INSERT INTO " + table + " (payload, topic) VALUES (?, ?)" //nolint:gosec + + txLower, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = txLower.Rollback() }) + + _, err = txLower.ExecContext(ctx, insertNotificationSQL, "lower", "topic") + require.NoError(t, err) + + insertCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + higherResult, err := dbPool.ExecContext(insertCtx, insertNotificationSQL, "higher", "topic") + require.NoError(t, err) + higherID, err := higherResult.LastInsertId() + require.NoError(t, err) + + connectFinished := testsignal.TestSignal[error]{} + connectFinished.Init(t) + connectStarted := testsignal.TestSignal[struct{}]{} + connectStarted.Init(t) + + go func() { + connectStarted.Signal(struct{}{}) + connectFinished.Signal(listener.Connect(ctx)) + }() + + connectStarted.WaitOrTimeout() + select { + case err := <-connectFinished.WaitC(): + require.NoError(t, err) + require.FailNow(t, "listener connected before the lower transaction committed") + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, txLower.Commit()) + require.NoError(t, connectFinished.WaitOrTimeout()) + require.Equal(t, higherID, listener.lastID) +} + +func TestListenerWaitForNotificationDoesNotSkipLowerUncommittedID(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + type waitForNotificationResult struct { + err error + found bool + notification *riverdriver.Notification + } + + var ( + ctx = t.Context() + dbPool = riversharedtest.DBPoolMySQL(ctx, t) + driver = New(dbPool) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + listener = driver.GetListener(&riverdriver.GetListenenerParams{Schema: schema}).(*Listener) //nolint:forcetypeassert + table = mysqlIdentifier(schema) + "." + mysqlIdentifier("river_notification") + ) + insertNotificationSQL := "INSERT INTO " + table + " (payload, topic) VALUES (?, ?)" //nolint:gosec + + listener.pollInterval = time.Millisecond + require.NoError(t, listener.Connect(ctx)) + t.Cleanup(func() { require.NoError(t, listener.Close(context.Background())) }) + require.NoError(t, listener.Listen(ctx, "topic")) + + txLower, err := dbPool.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = txLower.Rollback() }) + + _, err = txLower.ExecContext(ctx, insertNotificationSQL, "lower", "topic") + require.NoError(t, err) + + insertCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + _, err = dbPool.ExecContext(insertCtx, insertNotificationSQL, "higher", "topic") + require.NoError(t, err) + + started := testsignal.TestSignal[struct{}]{} + started.Init(t) + waitForNotification := testsignal.TestSignal[waitForNotificationResult]{} + waitForNotification.Init(t) + + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + started.Signal(struct{}{}) + + notification, found, err := listener.waitForNotificationOnce(waitCtx) + waitForNotification.Signal(waitForNotificationResult{ + err: err, + found: found, + notification: notification, + }) + }() + + started.WaitOrTimeout() + + select { + case res := <-waitForNotification.WaitC(): + require.NoError(t, res.err) + require.FailNow(t, "listener returned before lower transaction committed", "notification: %+v", res.notification) + case <-time.After(250 * time.Millisecond): + } + + require.NoError(t, txLower.Commit()) + + res := waitForNotification.WaitOrTimeout() + require.NoError(t, res.err) + require.True(t, res.found) + require.Equal(t, &riverdriver.Notification{Payload: "lower", Topic: "topic"}, res.notification) + + waitHigherCtx, cancel := context.WithTimeout(ctx, riversharedtest.WaitTimeout()) + defer cancel() + + notification, err := listener.WaitForNotification(waitHigherCtx) + require.NoError(t, err) + require.Equal(t, &riverdriver.Notification{Payload: "higher", Topic: "topic"}, notification) +} + +func TestQueueOperations(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + now := time.Now().UTC().Truncate(time.Microsecond) + + // Create a queue + queue, err := exec.QueueCreateOrSetUpdatedAt(ctx, &riverdriver.QueueCreateOrSetUpdatedAtParams{ + Metadata: []byte(`{}`), + Name: "test_queue", + Now: &now, + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test_queue", queue.Name) + require.Nil(t, queue.PausedAt) + + // Get the queue + fetched, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.Equal(t, "test_queue", fetched.Name) + + // List queues + queues, err := exec.QueueList(ctx, &riverdriver.QueueListParams{ + Max: 100, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, queues, 1) + + // Pause queue + err = exec.QueuePause(ctx, &riverdriver.QueuePauseParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + + paused, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.NotNil(t, paused.PausedAt) + + // Resume queue + err = exec.QueueResume(ctx, &riverdriver.QueueResumeParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + + resumed, err := exec.QueueGet(ctx, &riverdriver.QueueGetParams{ + Name: "test_queue", + Schema: schema, + }) + require.NoError(t, err) + require.Nil(t, resumed.PausedAt) +} + +func TestServerTimestampsAreUTC(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + execTx, err := driver.GetExecutor().Begin(ctx) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if closed { + return + } + _ = execTx.Exec(context.Background(), "SET time_zone = '+00:00'") + _ = execTx.Rollback(context.Background()) + }) + + require.NoError(t, execTx.Exec(ctx, "SET time_zone = '-05:00'")) + + beforeInsert := time.Now().UTC().Add(-time.Second) + job, err := execTx.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "server_timestamp_utc", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + afterInsert := time.Now().UTC().Add(time.Second) + + require.WithinRange(t, job.CreatedAt, beforeInsert, afterInsert) + require.WithinRange(t, job.ScheduledAt, beforeInsert, afterInsert) + + require.NoError(t, execTx.Exec(ctx, "SET time_zone = '+00:00'")) + require.NoError(t, execTx.Rollback(ctx)) + closed = true +} + +func TestTimeParametersAreNormalizedToUTC(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + localTime := time.Date(2026, time.July, 24, 12, 34, 56, 123456000, time.FixedZone("UTC-7", -7*60*60)) + job, err := driver.GetExecutor().JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + CreatedAt: &localTime, + EncodedArgs: []byte(`{}`), + Kind: "time_parameters_utc", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + ScheduledAt: &localTime, + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + require.Equal(t, localTime.UTC(), job.CreatedAt) + require.Equal(t, localTime.UTC(), job.ScheduledAt) +} + +func TestTransactionReadsCommittedWrites(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + ctx := t.Context() + driver := New(riversharedtest.DBPoolMySQL(ctx, t)) + schema := riverdbtest.TestSchema(ctx, t, driver, nil) + exec := driver.GetExecutor() + tx, err := exec.Begin(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback(context.Background()) }) + + countParams := &riverdriver.JobCountByStateParams{Schema: schema, State: rivertype.JobStateAvailable} + count, err := tx.JobCountByState(ctx, countParams) + require.NoError(t, err) + require.Zero(t, count) + + _, err = exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), Kind: "committed_after_begin", MaxAttempts: 3, + Priority: 1, Queue: "default", Schema: schema, State: rivertype.JobStateAvailable, Tags: []string{}, + }) + require.NoError(t, err) + + count, err = tx.JobCountByState(ctx, countParams) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestTransactions(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Begin and commit + tx, err := exec.Begin(ctx) + require.NoError(t, err) + + _, err = tx.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "tx_test", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + require.NoError(t, tx.Commit(ctx)) + + // Verify job exists + count, err := exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 1, count) + + // Begin and rollback + tx2, err := exec.Begin(ctx) + require.NoError(t, err) + + _, err = tx2.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "tx_test_rollback", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + require.NoError(t, tx2.Rollback(ctx)) + + // Verify second job was rolled back + count, err = exec.JobCountByState(ctx, &riverdriver.JobCountByStateParams{ + Schema: schema, + State: rivertype.JobStateAvailable, + }) + require.NoError(t, err) + require.Equal(t, 1, count) // still 1 +} + +func TestJobInsertFastMany(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + results, err := exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{ + { + EncodedArgs: []byte(`{"i": 1}`), + Kind: "fast_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + { + EncodedArgs: []byte(`{"i": 2}`), + Kind: "fast_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + State: rivertype.JobStateAvailable, + Tags: []string{}, + }, + }, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 2) + + for _, result := range results { + require.NotNil(t, result.Job) + require.False(t, result.UniqueSkippedAsDuplicate) + } +} + +func TestJobMetadata(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a job with metadata + job, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "metadata_test", + MaxAttempts: 3, + Metadata: []byte(`{"key": "value"}`), + Priority: 1, + Queue: "default", + Schema: schema, + State: rivertype.JobStateAvailable, + Tags: []string{}, + }) + require.NoError(t, err) + + var metadata map[string]any + require.NoError(t, json.Unmarshal(job.Metadata, &metadata)) + require.Equal(t, "value", metadata["key"]) + + // Update metadata + updated, err := exec.JobUpdate(ctx, &riverdriver.JobUpdateParams{ + ID: job.ID, + MetadataDoMerge: true, + Metadata: []byte(`{"new_key": "new_value"}`), + Schema: schema, + }) + require.NoError(t, err) + + require.NoError(t, json.Unmarshal(updated.Metadata, &metadata)) + require.Equal(t, "value", metadata["key"]) + require.Equal(t, "new_value", metadata["new_key"]) +} + +func TestJobSchedule(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + exec = driver.GetExecutor() + ) + + // Insert a scheduled job with a past scheduled_at + past := time.Now().UTC().Add(-1 * time.Hour).Truncate(time.Microsecond) + _, err := exec.JobInsertFull(ctx, &riverdriver.JobInsertFullParams{ + EncodedArgs: []byte(`{}`), + Kind: "scheduled_job", + MaxAttempts: 3, + Priority: 1, + Queue: "default", + ScheduledAt: &past, + Schema: schema, + State: rivertype.JobStateScheduled, + Tags: []string{}, + }) + require.NoError(t, err) + + // Schedule should find and transition the job + now := time.Now().UTC() + results, err := exec.JobSchedule(ctx, &riverdriver.JobScheduleParams{ + Max: 100, + Now: &now, + Schema: schema, + }) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, rivertype.JobStateAvailable, results[0].Job.State) +} + +func TestNotifyMany(t *testing.T) { + t.Parallel() + + driver := New(nil) + require.NotNil(t, driver.GetListener(&riverdriver.GetListenenerParams{})) +} + +func TestNotifyManyInsertsNotifications(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + driver = New(riversharedtest.DBPoolMySQL(ctx, t)) + exec = driver.GetExecutor() + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + ) + + payloads := []string{"test1", "", "é🌊\n\"quoted\"", "null", "test2"} + err := exec.NotifyMany(ctx, &riverdriver.NotifyManyParams{ + Payload: payloads, + Schema: schema, + Topic: "test_topic", + }) + require.NoError(t, err) + + rows, err := driver.dbPool.QueryContext(ctx, "SELECT payload, topic FROM "+mysqlIdentifier(schema)+".river_notification ORDER BY id") //nolint:gosec + require.NoError(t, err) + defer rows.Close() + var received []string + for rows.Next() { + var payload, topic string + require.NoError(t, rows.Scan(&payload, &topic)) + require.Equal(t, "test_topic", topic) + received = append(received, payload) + } + require.NoError(t, rows.Err()) + require.Equal(t, payloads, received) +} + +func TestPGAdvisoryXactLock(t *testing.T) { + t.Parallel() + + riversharedtest.SkipIfMySQLNotEnabled(t) + + var ( + ctx = t.Context() + exec = New(riversharedtest.DBPoolMySQL(ctx, t)).GetExecutor() + ) + + _, err := exec.PGAdvisoryXactLock(ctx, 12345) + require.ErrorIs(t, err, riverdriver.ErrNotImplemented) +} diff --git a/riverdriver/rivermysql/river_mysql_listener.go b/riverdriver/rivermysql/river_mysql_listener.go new file mode 100644 index 000000000..cd48c6103 --- /dev/null +++ b/riverdriver/rivermysql/river_mysql_listener.go @@ -0,0 +1,316 @@ +package rivermysql + +import ( + "context" + "database/sql" + "errors" + "sync" + "time" + + "github.com/riverqueue/river/riverdriver" + "github.com/riverqueue/river/riverdriver/rivermysql/internal/dbsqlc" + "github.com/riverqueue/river/rivershared/sqlctemplate" +) + +const ( + notificationBatchSize = 256 + notificationPollIntervalDefault = 50 * time.Millisecond +) + +// Listener receives MySQL notifications from the river_notification outbox +// table. MySQL doesn't have a native LISTEN/NOTIFY equivalent, so NotifyMany +// appends rows to river_notification and this listener polls for rows with IDs +// greater than its remembered lastID. Polls use a locking read because InnoDB +// may commit auto-increment IDs out of order across concurrent transactions; a +// normal consistent read could otherwise skip a lower uncommitted notification. +type Listener struct { + afterConnectExec string // should only ever be used in testing + dbPool *sql.DB + isConnected bool + lastID int64 + mu sync.Mutex + pending []*dbsqlc.RiverNotification + pollInterval time.Duration + replacer *sqlctemplate.Replacer + schema string + topics map[string]int64 +} + +func (l *Listener) Close(context.Context) error { + l.mu.Lock() + defer l.mu.Unlock() + + l.isConnected = false + l.pending = nil + l.topics = nil + return nil +} + +func (l *Listener) Connect(ctx context.Context) error { + var ( + afterConnectExec string + dbPool *sql.DB + replacer *sqlctemplate.Replacer + schema string + ) + + l.mu.Lock() + if l.isConnected { + l.mu.Unlock() + return errors.New("connection already established") + } + afterConnectExec = l.afterConnectExec + dbPool = l.dbPool + replacer = l.replacer + schema = l.schema + l.mu.Unlock() + + if dbPool == nil { + return errors.New("database pool is nil") + } + if replacer == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } + + if afterConnectExec != "" { + if _, err := dbPool.ExecContext(ctx, afterConnectExec); err != nil { + return err + } + } + + // Establish the initial high-water mark with a locking scan. InnoDB allocates + // auto-increment IDs before commit, so a higher ID can commit while a lower + // ID is still uncommitted. A normal MAX(id) read would advance past the lower + // row and permanently skip it when it commits. Scan the IDs in ascending + // order with FOR UPDATE so the read reaches and waits for every lower + // uncommitted row before establishing the high-water mark. In particular, + // this must not be reduced to an aggregate MAX or descending LIMIT query, + // neither of which has to encounter the lower row. + tx, err := dbPool.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + ids, err := dbsqlc.New().NotificationGetIDsForUpdate(schemaTemplateParam(ctx, schema), notificationDBTX(tx, replacer)) + if err != nil { + return err + } + var lastID int64 + if len(ids) > 0 { + lastID = ids[len(ids)-1] + } + if err := tx.Commit(); err != nil { + return err + } + + l.mu.Lock() + defer l.mu.Unlock() + + if l.isConnected { + return errors.New("connection already established") + } + + l.isConnected = true + l.lastID = lastID + + return nil +} + +func (l *Listener) Listen(ctx context.Context, topic string) error { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return errors.New("listener is not connected") + } + + if _, ok := l.topics[topic]; ok { + return nil + } + + // Establish a starting ID for this topic without skipping pending messages + // for existing subscriptions. As in Connect, scan in ascending order with + // locks to account for notifications that haven't committed yet. + tx, err := l.dbPool.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + ids, err := dbsqlc.New().NotificationGetIDsForUpdate(schemaTemplateParam(ctx, l.schema), notificationDBTX(tx, l.replacer)) + if err != nil { + return err + } + lastID := l.lastID + if len(ids) > 0 { + lastID = max(lastID, ids[len(ids)-1]) + } + if err := tx.Commit(); err != nil { + return err + } + + if l.topics == nil { + l.topics = make(map[string]int64) + } + + l.topics[topic] = lastID + return nil +} + +func (l *Listener) Ping(ctx context.Context) error { + dbPool, err := l.stateDBPool() + if err != nil { + return err + } + return dbPool.PingContext(ctx) +} + +func (l *Listener) Schema() string { + l.mu.Lock() + defer l.mu.Unlock() + + return l.schema +} + +func (l *Listener) SetAfterConnectExec(sql string) { + l.mu.Lock() + defer l.mu.Unlock() + + l.afterConnectExec = sql +} + +func (l *Listener) Unlisten(_ context.Context, topic string) error { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return errors.New("listener is not connected") + } + + delete(l.topics, topic) + return nil +} + +func (l *Listener) WaitForNotification(ctx context.Context) (*riverdriver.Notification, error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + + notification, found, err := l.waitForNotificationOnce(ctx) + if errors.Is(err, sql.ErrNoRows) { + if err := l.waitForNextPoll(ctx); err != nil { + return nil, err + } + continue + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, err + } + if found { + return notification, nil + } + } +} + +func (l *Listener) stateDBPool() (*sql.DB, error) { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return nil, errors.New("listener is not connected") + } + if l.dbPool == nil { + return nil, errors.New("database pool is nil") + } + + return l.dbPool, nil +} + +func (l *Listener) waitForNextPoll(ctx context.Context) error { + l.mu.Lock() + pollInterval := l.pollInterval + l.mu.Unlock() + + if pollInterval <= 0 { + pollInterval = notificationPollIntervalDefault + } + + timer := time.NewTimer(pollInterval) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (l *Listener) waitForNotificationOnce(ctx context.Context) (*riverdriver.Notification, bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.isConnected { + return nil, false, errors.New("listener is not connected") + } + if l.dbPool == nil { + return nil, false, errors.New("database pool is nil") + } + + if len(l.pending) == 0 { + if len(l.topics) == 0 { + return nil, false, sql.ErrNoRows + } + tx, err := l.dbPool.BeginTx(ctx, nil) + if err != nil { + return nil, false, err + } + defer tx.Rollback() + + // Preserve ascending locking reads: a lower ID might still be + // uncommitted. Shared locks allow other listeners to poll concurrently. + // Read all topics so this scan follows the primary key's order. + notifications, err := dbsqlc.New().NotificationGetAfter( + schemaTemplateParam(ctx, l.schema), notificationDBTX(tx, l.replacer), + &dbsqlc.NotificationGetAfterParams{After: l.lastID, Limit: notificationBatchSize}, + ) + if err != nil { + return nil, false, err + } + if err := tx.Commit(); err != nil { + return nil, false, err + } + if len(notifications) == 0 { + return nil, false, sql.ErrNoRows + } + l.pending = notifications + l.lastID = notifications[len(notifications)-1].ID + } + + for len(l.pending) > 0 { + notification := l.pending[0] + l.pending[0] = nil + l.pending = l.pending[1:] + + // Subscriptions may change after fetching a batch but before delivery. + if startID, ok := l.topics[notification.Topic]; ok && notification.ID > startID { + return &riverdriver.Notification{ + Payload: notification.Payload, + Topic: notification.Topic, + }, true, nil + } + } + return nil, false, nil +} + +func notificationDBTX(dbtx dbsqlc.DBTX, replacer *sqlctemplate.Replacer) templateReplaceWrapper { + if replacer == nil { + replacer = &sqlctemplate.Replacer{UnnumberedPlaceholders: true} + } + return templateReplaceWrapper{dbtx, replacer} +} diff --git a/riverdriver/riverpgxv5/go.sum b/riverdriver/riverpgxv5/go.sum index a19a74c71..cc1106e35 100644 --- a/riverdriver/riverpgxv5/go.sum +++ b/riverdriver/riverpgxv5/go.sum @@ -8,14 +8,14 @@ github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QII github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql index cc921c261..79599df41 100644 --- a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql +++ b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql @@ -651,7 +651,9 @@ WITH job_input AS ( unnest(@finalized_at_do_update::boolean[]) AS finalized_at_do_update, unnest(@finalized_at::timestamptz[]) AS finalized_at, unnest(@metadata_do_merge::boolean[]) AS metadata_do_merge, - unnest(@metadata_updates::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest(@metadata_updates::text[])::jsonb AS metadata_updates, unnest(@scheduled_at_do_update::boolean[]) AS scheduled_at_do_update, unnest(@scheduled_at::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] diff --git a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go index 165b85af0..d78af9d5e 100644 --- a/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go +++ b/riverdriver/riverpgxv5/internal/dbsqlc/river_job.sql.go @@ -1485,7 +1485,9 @@ WITH job_input AS ( unnest($6::boolean[]) AS finalized_at_do_update, unnest($7::timestamptz[]) AS finalized_at, unnest($8::boolean[]) AS metadata_do_merge, - unnest($9::jsonb[]) AS metadata_updates, + -- Cast text elements on the server so database/sql drivers do not decode + -- JSON numbers through float64 while converting array parameters. + unnest($9::text[])::jsonb AS metadata_updates, unnest($10::boolean[]) AS scheduled_at_do_update, unnest($11::timestamptz[]) AS scheduled_at, -- To avoid requiring pgx users to register the OID of the river_job_state[] @@ -1565,7 +1567,7 @@ type JobSetStateIfRunningManyParams struct { FinalizedAtDoUpdate []bool FinalizedAt []time.Time MetadataDoMerge []bool - MetadataUpdates [][]byte + MetadataUpdates []string ScheduledAtDoUpdate []bool ScheduledAt []time.Time State []string diff --git a/riverdriver/riverpgxv5/river_pgx_v5_driver.go b/riverdriver/riverpgxv5/river_pgx_v5_driver.go index 71d56fef6..572efaf6d 100644 --- a/riverdriver/riverpgxv5/river_pgx_v5_driver.go +++ b/riverdriver/riverpgxv5/river_pgx_v5_driver.go @@ -65,8 +65,9 @@ func New(dbPool *pgxpool.Pool) *Driver { const argPlaceholder = "$" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNamePostgres } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{templateReplaceWrapper{d.dbPool, &d.replacer}, d} @@ -103,6 +104,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] return fmt.Sprintf("%s && @%s", column, namedArg), values, nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return column + " @> @" + namedArg + "::jsonb", map[string]any{namedArg: fragment}, nil +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { return fmt.Sprintf("%s = any(@%s)", column, column), values, nil } @@ -687,7 +692,7 @@ func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdr FinalizedAt: make([]time.Time, len(params.ID)), FinalizedAtDoUpdate: make([]bool, len(params.ID)), MetadataDoMerge: make([]bool, len(params.ID)), - MetadataUpdates: make([][]byte, len(params.ID)), + MetadataUpdates: make([]string, len(params.ID)), Now: params.Now, ScheduledAt: make([]time.Time, len(params.ID)), ScheduledAtDoUpdate: make([]bool, len(params.ID)), @@ -708,7 +713,9 @@ func (e *Executor) JobSetStateIfRunningMany(ctx context.Context, params *riverdr } if params.MetadataDoMerge[i] { setStateParams.MetadataDoMerge[i] = true - setStateParams.MetadataUpdates[i] = params.MetadataUpdates[i] + setStateParams.MetadataUpdates[i] = string(params.MetadataUpdates[i]) + } else { + setStateParams.MetadataUpdates[i] = "{}" } if params.ScheduledAt[i] != nil { setStateParams.ScheduledAtDoUpdate[i] = true diff --git a/riverdriver/riversqlite/go.sum b/riverdriver/riversqlite/go.sum index c89f3a216..470027c2a 100644 --- a/riverdriver/riversqlite/go.sum +++ b/riverdriver/riversqlite/go.sum @@ -6,16 +6,16 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/rivershared v0.51.0 h1:R5MINLt1LNMdixuyzvXfhU7HANDWSKMSQy+D7PpvHz8= -github.com/riverqueue/river/rivershared v0.51.0/go.mod h1:MECL7bG8KLo2hMGFpnPeVAzMUZO17pu6OxG26E0h9yU= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivershared v0.51.1 h1:ag7playa5HkdLKUFukeV84OTz13ErZrQQjR2U7M85aY= +github.com/riverqueue/river/rivershared v0.51.1/go.mod h1:VHm5KZm0Cdv4CilyBq/Xd2tx7xnaAmo1zCA+TjVffU8= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/riverdriver/riversqlite/river_sqlite_driver.go b/riverdriver/riversqlite/river_sqlite_driver.go index 7d1044bfc..57019cdc0 100644 --- a/riverdriver/riversqlite/river_sqlite_driver.go +++ b/riverdriver/riversqlite/river_sqlite_driver.go @@ -73,8 +73,9 @@ func New(dbPool *sql.DB) *Driver { const argPlaceholder = "?" -func (d *Driver) ArgPlaceholder() string { return argPlaceholder } -func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameSQLite } +func (d *Driver) ArgPlaceholder() string { return argPlaceholder } +func (d *Driver) DatabaseName() string { return riverdriver.DatabaseNameSQLite } +func (d *Driver) SafeIdentifier(ident string) string { return dbutil.SafeIdentifier(ident) } func (d *Driver) GetExecutor() riverdriver.Executor { return &Executor{d.dbPool, templateReplaceWrapper{d.dbPool, &d.replacer}, d, nil} @@ -146,6 +147,10 @@ func (d *Driver) SQLFragmentColumnContainsAny(column, namedArg string, values [] )`, column, namedArg), arg, nil } +func (d *Driver) SQLFragmentColumnContainsJSON(column, namedArg, fragment string) (string, map[string]any, error) { + return "", nil, errors.New("JSON containment is not supported on SQLite") +} + func (d *Driver) SQLFragmentColumnIn(column string, values any) (string, any, error) { arg, err := json.Marshal(values) if err != nil { diff --git a/rivermigrate/river_migrate.go b/rivermigrate/river_migrate.go index a7f9d1552..ef49ed847 100644 --- a/rivermigrate/river_migrate.go +++ b/rivermigrate/river_migrate.go @@ -339,11 +339,17 @@ func (m *Migrator[TTx]) Migrate(ctx context.Context, direction Direction, opts * // // This variant lets a caller run migrations within a transaction. Postgres DDL // is transactional, so migration changes aren't visible until the transaction -// commits, and are rolled back if the transaction rolls back. +// commits, and are rolled back if the transaction rolls back. MySQL rejects this +// method because its DDL implicitly commits the caller's transaction. Use Migrate +// instead. // // Deprecated: Use Migrate instead. Certain migrations cannot be batched together // in a single transaction, so this method is not recommended. func (m *Migrator[TTx]) MigrateTx(ctx context.Context, tx TTx, direction Direction, opts *MigrateOpts) (*MigrateResult, error) { + if m.driver.DatabaseName() == riverdriver.DatabaseNameMySQL { + return nil, errors.New("MySQL migrations cannot run in a transaction because DDL commits implicitly; use Migrate instead") + } + switch direction { case DirectionDown: return m.migrateDown(ctx, m.driver.UnwrapExecutor(tx), direction, opts, true) @@ -583,7 +589,7 @@ func (m *Migrator[TTx]) applyMigrations(ctx context.Context, exec riverdriver.Ex var schema string if m.schema != "" { - schema = dbutil.SafeIdentifier(m.schema) + "." + schema = m.driver.SafeIdentifier(m.schema) + "." } schemaReplacement := map[string]sqlctemplate.Replacement{ "schema": {Value: schema}, diff --git a/rivershared/go.sum b/rivershared/go.sum index 1dfbd8181..e91da7341 100644 --- a/rivershared/go.sum +++ b/rivershared/go.sum @@ -10,14 +10,14 @@ github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QII github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao= github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/riverqueue/river v0.51.0 h1:Kc1hqdVCrJOwYJx5K5Tv4IS9ykadGzUAGN8X2TfYSUc= -github.com/riverqueue/river v0.51.0/go.mod h1:n989vrgdCVGIJMsRyS3dykJe3zuUcgm2fAh82glojGY= -github.com/riverqueue/river/riverdriver v0.51.0 h1:UWXnG5TmOoFWRLkAupbg12/yUHtk/vhMre/U6gNYqoc= -github.com/riverqueue/river/riverdriver v0.51.0/go.mod h1:tuoOQmuQUvWPrhwBNkFsqrAkZrg6Cpbk9aOF8mqhV/M= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0 h1:+smCM+3oYwwcSclO62nt0+ub89/E6HE1B/sEO2IRx7o= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.0/go.mod h1:krkXW2gE5y8fyEinO0t9Ch89fBnDLJJezOrdKo4jo9Y= -github.com/riverqueue/river/rivertype v0.51.0 h1:0alGpHNfFxWQMF3HEXixPO+ezG5FiOO/E9GraAo8CPw= -github.com/riverqueue/river/rivertype v0.51.0/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= +github.com/riverqueue/river v0.51.1 h1:jDDXN0lMBpDcNIr02uDXXW6qqantVOO5k/MDQvZAlJM= +github.com/riverqueue/river v0.51.1/go.mod h1:ORGKNgZPGpT32m0DZ/QN+1Py8E4xua7fzxSYLEtQFQQ= +github.com/riverqueue/river/riverdriver v0.51.1 h1:Rp/XZMDCiULsAsUG+2NMDYjLo7Awd2Miy6X8qy3UPkI= +github.com/riverqueue/river/riverdriver v0.51.1/go.mod h1:2iztjGOsU5EgcW+nX6OK8eic+itlm7cnS5+31QArANQ= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1 h1:ojQHIvNGa7YRKjoCN5OC/N/vxKWwmVco761q2RozI+U= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.51.1/go.mod h1:9WSLa2Sirz4B8WbKB8nEOZh6mfW3tA1OT9VgMKlIc0E= +github.com/riverqueue/river/rivertype v0.51.1 h1:iV6EdBW2OHRwZQAnnbwZwAvDAY8Vy0PghrEpQGjjFhw= +github.com/riverqueue/river/rivertype v0.51.1/go.mod h1:XKkcRQR6zm8RR/JQa1Q2ywpj8uXQu21quPa4Lpw1Xhw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/rivershared/riversharedtest/riversharedtest.go b/rivershared/riversharedtest/riversharedtest.go index a02143ade..662354846 100644 --- a/rivershared/riversharedtest/riversharedtest.go +++ b/rivershared/riversharedtest/riversharedtest.go @@ -112,6 +112,47 @@ var sqliteTestDir = sync.OnceValue(func() string { //nolint:gochecknoglobals return path.Join(rootDir, "sqlite") }) +// A pool and sync.Once to initialize a MySQL pool, invoked by DBPoolMySQL. +var ( + dbPoolMySQL *sql.DB //nolint:gochecknoglobals + dbPoolMySQLOnce sync.Once //nolint:gochecknoglobals + errDBPoolMySQL error //nolint:gochecknoglobals +) + +// SkipIfMySQLNotEnabled skips the current test if MySQL tests are not enabled. +// MySQL tests are opt-in because they require a running MySQL server. Set +// RIVER_MYSQL_TESTS_ENABLED=1 or RIVER_MYSQL_TESTS_ENABLED=true to enable. +func SkipIfMySQLNotEnabled(tb testing.TB) { + tb.Helper() + + val := os.Getenv("RIVER_MYSQL_TESTS_ENABLED") + if val != "1" && val != "true" { //nolint:goconst + tb.Skip("Skipping MySQL tests; set RIVER_MYSQL_TESTS_ENABLED=1 to enable") + } +} + +// DBPoolMySQL gets a lazily initialized database pool for MySQL testing. +// Uses TEST_MYSQL_URL or defaults to root@tcp(localhost:3306)/?. +func DBPoolMySQL(ctx context.Context, tb testing.TB) *sql.DB { + tb.Helper() + + dbPoolMySQLOnce.Do(func() { + dsn := cmp.Or( + os.Getenv("TEST_MYSQL_URL"), + "root@tcp(localhost:3306)/?parseTime=true&multiStatements=true&loc=UTC&time_zone=%27%2B00%3A00%27", + ) + + dbPoolMySQL, errDBPoolMySQL = sql.Open("mysql", dsn) + if errDBPoolMySQL == nil { + errDBPoolMySQL = dbPoolMySQL.PingContext(ctx) + } + }) + require.NoError(tb, errDBPoolMySQL) + require.NotNil(tb, dbPoolMySQL) + + return dbPoolMySQL +} + // DBPoolLibSQL gets a database pool appropriate for use with libSQL (a SQLite // fork) in testing. func DBPoolLibSQL(ctx context.Context, tb testing.TB, schema string) *sql.DB { diff --git a/rivershared/sqlctemplate/sqlc_template.go b/rivershared/sqlctemplate/sqlc_template.go index 1151b7d13..86fc1e115 100644 --- a/rivershared/sqlctemplate/sqlc_template.go +++ b/rivershared/sqlctemplate/sqlc_template.go @@ -89,6 +89,14 @@ type Replacement struct { // be initialized with a constructor. This lets it default to a usable instance // on drivers that may themselves not be initialized. type Replacer struct { + // UnnumberedPlaceholders, when true, causes Run to emit plain `?` + // placeholders instead of numbered `?1`, `?2`, etc. for named args + // injected via the template system. The args slice is reordered to + // match the positional order of placeholders in the SQL. This is + // needed for MySQL, whose database/sql driver does not support + // numbered `?N` syntax. + UnnumberedPlaceholders bool + cache map[replacerCacheKey]string cacheMu sync.RWMutex } @@ -142,6 +150,13 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar } cacheKey, cacheEligible := replacerCacheKeyFrom(sql, container) + + // In unnumbered mode, named args are interleaved with sqlc args during a + // left-to-right SQL walk. The cache can't reconstruct this ordering on hit. + if r.UnnumberedPlaceholders && len(container.NamedArgs) > 0 { + cacheEligible = false + } + if cacheEligible { r.cacheMu.RLock() var ( @@ -212,28 +227,36 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar } if len(container.NamedArgs) > 0 { - placeholderNum := len(args) - // For the benefit of the test suite's output being predictable, sort // named args before processing them. sortedNamedArgs := maputil.Keys(container.NamedArgs) slices.Sort(sortedNamedArgs) - for _, arg := range sortedNamedArgs { - placeholderNum++ - - var ( - symbol = "@" + arg - symbolIndex = strings.Index(updatedSQL, symbol) - val = container.NamedArgs[arg] - ) - if symbolIndex == -1 { - return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", symbol) + if r.UnnumberedPlaceholders { + var err error + updatedSQL, args, err = replaceUnnumberedArgs(updatedSQL, args, container.NamedArgs, sortedNamedArgs) + if err != nil { + return "", nil, err } + } else { + placeholderNum := len(args) + for _, arg := range sortedNamedArgs { + placeholderNum++ + + var ( + symbol = "@" + arg + symbolIndex = strings.Index(updatedSQL, symbol) + val = container.NamedArgs[arg] + ) + + if symbolIndex == -1 { + return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", symbol) + } - // ReplaceAll because an input parameter may appear multiple times. - updatedSQL = strings.ReplaceAll(updatedSQL, symbol, argPlaceholder+strconv.Itoa(placeholderNum)) - args = append(args, val) + // ReplaceAll because an input parameter may appear multiple times. + updatedSQL = strings.ReplaceAll(updatedSQL, symbol, argPlaceholder+strconv.Itoa(placeholderNum)) + args = append(args, val) + } } } @@ -249,6 +272,148 @@ func (r *Replacer) RunSafely(ctx context.Context, argPlaceholder, sql string, ar return updatedSQL, args, nil } +func replaceUnnumberedArgs(sql string, args []any, namedArgs map[string]any, sortedNamedArgs []string) (string, []any, error) { + // Prefer the longest name when one named argument is a prefix of another. + slices.SortFunc(sortedNamedArgs, func(left, right string) int { + switch { + case len(left) > len(right): + return -1 + case len(left) < len(right): + return 1 + default: + return strings.Compare(left, right) + } + }) + + var ( + inBlockComment bool + inLineComment bool + newArgs = make([]any, 0, len(args)+len(namedArgs)) + out strings.Builder + quote byte + sqlcArgIndex int + usedNamedArgs = make(map[string]bool, len(namedArgs)) + ) + out.Grow(len(sql)) + + for i := 0; i < len(sql); { + if inLineComment { + out.WriteByte(sql[i]) + if sql[i] == '\n' { + inLineComment = false + } + i++ + continue + } + + if inBlockComment { + if sql[i] == '*' && i+1 < len(sql) && sql[i+1] == '/' { + out.WriteString("*/") + i += 2 + inBlockComment = false + continue + } + out.WriteByte(sql[i]) + i++ + continue + } + + if quote != 0 { + out.WriteByte(sql[i]) + if sql[i] == '\\' && i+1 < len(sql) { + out.WriteByte(sql[i+1]) + i += 2 + continue + } + if sql[i] == quote { + if i+1 < len(sql) && sql[i+1] == quote { + out.WriteByte(sql[i+1]) + i += 2 + continue + } + quote = 0 + } + i++ + continue + } + + switch { + case sql[i] == '\'', sql[i] == '"', sql[i] == '`': + quote = sql[i] + out.WriteByte(sql[i]) + i++ + + case sql[i] == '#': + inLineComment = true + out.WriteByte(sql[i]) + i++ + + case sql[i] == '-' && i+2 < len(sql) && sql[i+1] == '-' && sql[i+2] <= ' ': + inLineComment = true + out.WriteString("--") + i += 2 + + case sql[i] == '/' && i+1 < len(sql) && sql[i+1] == '*': + inBlockComment = true + out.WriteString("/*") + i += 2 + + case sql[i] == '@': + matched := false + for _, name := range sortedNamedArgs { + symbol := "@" + name + if !strings.HasPrefix(sql[i:], symbol) { + continue + } + + end := i + len(symbol) + if end < len(sql) && isNamedArgChar(sql[end]) { + continue + } + + out.WriteByte('?') + newArgs = append(newArgs, namedArgs[name]) + usedNamedArgs[name] = true + i = end + matched = true + break + } + if !matched { + out.WriteByte(sql[i]) + i++ + } + + case sql[i] == '?': + if sqlcArgIndex >= len(args) { + return "", nil, errors.New("sqlctemplate found more unnumbered placeholders than sqlc arguments") + } + out.WriteByte('?') + newArgs = append(newArgs, args[sqlcArgIndex]) + sqlcArgIndex++ + i++ + + default: + out.WriteByte(sql[i]) + i++ + } + } + + if sqlcArgIndex != len(args) { + return "", nil, errors.New("sqlctemplate found fewer unnumbered placeholders than sqlc arguments") + } + for _, name := range sortedNamedArgs { + if !usedNamedArgs[name] { + return "", nil, fmt.Errorf("sqltemplate expected to find named arg %q, but it wasn't present", "@"+name) + } + } + + return out.String(), newArgs, nil +} + +func isNamedArgChar(c byte) bool { + return c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' +} + // WithReplacements adds sqlctemplate templates to the given context (they go in // context because it's the only way to get them down into the innards of sqlc). // namedArgs can also be passed in to replace arguments found in diff --git a/rivershared/sqlctemplate/sqlc_template_test.go b/rivershared/sqlctemplate/sqlc_template_test.go index 119ebd546..5441b26cd 100644 --- a/rivershared/sqlctemplate/sqlc_template_test.go +++ b/rivershared/sqlctemplate/sqlc_template_test.go @@ -259,7 +259,7 @@ func TestReplacer(t *testing.T) { AND state = @state; ` - // Initially cached value + // Initially cached value. { ctx := WithReplacements(ctx, map[string]Replacement{ "schema": {Stable: true, Value: "test_schema."}, @@ -420,6 +420,254 @@ func TestReplacer(t *testing.T) { `, updatedSQL) }) + t.Run("UnnumberedPlaceholders_NoNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Value: "test_schema."}, + }, nil) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE state = ?; + `, []any{"available"}) + require.NoError(t, err) + require.Equal(t, []any{"available"}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE state = ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgsNoInitialArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */; + `, nil) + require.NoError(t, err) + require.Equal(t, []any{"no_op"}, args) + require.Equal(t, ` + SELECT count(*) + FROM river_job + WHERE kind = ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgsWithInitialArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + // The named arg @kind appears in the WHERE clause before the + // sqlc-generated ? for LIMIT. UnnumberedPlaceholders reorders args so + // that they match the positional ? order in the final SQL. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NamedArgRepeated", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind OR queue = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + // When a named arg appears multiple times, it should produce a ? for + // each occurrence with the value duplicated in the args slice. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT * + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", "no_op", 100}, args) + require.Equal(t, ` + SELECT * + FROM river_job + WHERE kind = ? OR queue = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_MultipleNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + "where_clause": {Value: "kind = @kind AND status = @status"}, + }, map[string]any{ + "kind": "no_op", + "status": "succeeded", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", "succeeded", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? AND status = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_QuotedAndCommentedSymbols", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "where_clause": {Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + updatedSQL, args, err := replacer.RunSafely(ctx, "?", ` + SELECT '?' AS question, '@kind' AS literal + FROM river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + /* ? @kind */ + LIMIT ?; + `, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT '?' AS question, '@kind' AS literal + FROM river_job + WHERE kind = ? + /* ? @kind */ + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_NotCachedWithNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + "where_clause": {Stable: true, Value: "kind = @kind"}, + }, map[string]any{ + "kind": "no_op", + }) + + sql := ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + WHERE /* TEMPLATE_BEGIN: where_clause */ true /* TEMPLATE_END */ + LIMIT ?; + ` + + // Unnumbered mode with named args skips caching because the + // cached SQL can't preserve the positional arg ordering. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", sql, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + + require.Empty(t, replacer.cache) + + // Second call still produces correct results. + updatedSQL, args, err = replacer.RunSafely(ctx, "?", sql, []any{200}) + require.NoError(t, err) + require.Equal(t, []any{"no_op", 200}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + WHERE kind = ? + LIMIT ?; + `, updatedSQL) + }) + + t.Run("UnnumberedPlaceholders_CachedWithoutNamedArgs", func(t *testing.T) { + t.Parallel() + + replacer := &Replacer{UnnumberedPlaceholders: true} + + ctx := WithReplacements(ctx, map[string]Replacement{ + "schema": {Stable: true, Value: "test_schema."}, + }, nil) + + sql := ` + SELECT count(*) + FROM /* TEMPLATE: schema */river_job + LIMIT ?; + ` + + // Without named args, caching works normally in unnumbered mode. + updatedSQL, args, err := replacer.RunSafely(ctx, "?", sql, []any{100}) + require.NoError(t, err) + require.Equal(t, []any{100}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + LIMIT ?; + `, updatedSQL) + + require.Len(t, replacer.cache, 1) + + // Second call uses cache. + updatedSQL, args, err = replacer.RunSafely(ctx, "?", sql, []any{200}) + require.NoError(t, err) + require.Equal(t, []any{200}, args) + require.Equal(t, ` + SELECT count(*) + FROM test_schema.river_job + LIMIT ?; + `, updatedSQL) + }) + t.Run("Stress", func(t *testing.T) { t.Parallel() diff --git a/ruby/CHANGELOG.md b/ruby/CHANGELOG.md index 02511619f..082dc9d53 100644 --- a/ruby/CHANGELOG.md +++ b/ruby/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add MySQL support to the Active Record and Sequel drivers, including worker execution, transactional job operations, and canonical migrations. + ## [0.16.0] - 2026-11-04 ### Added diff --git a/ruby/Gemfile b/ruby/Gemfile index 609c5bf3a..2a7dc25b7 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -11,6 +11,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "debug" gem "fugit", "~> 1.14", require: false gem "minitest", require: false diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index ba520ba8b..59cad9122 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -85,6 +85,8 @@ GEM drb (~> 2.0) prism (~> 1.5) mutex_m (0.3.0) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.28.0) parser (3.3.13.0) @@ -197,6 +199,7 @@ DEPENDENCIES fugit (~> 1.14) json (< 3) minitest + mysql2 (~> 0.5) pg riverqueue! riverqueue-activerecord! diff --git a/ruby/Makefile b/ruby/Makefile index f6d142fb0..afc31d6b0 100644 --- a/ruby/Makefile +++ b/ruby/Makefile @@ -7,7 +7,7 @@ TEST_JOBS ?= 4 generate/fixtures: $(MAKE) -C .. generate/fixtures -test/conformance: generate/fixtures ## Run the Go-generated fixture checks without Postgres +test/conformance: generate/fixtures ## Run Go fixture checks (SQLite by default; TEST_DATABASE=mysql for MySQL) bundle exec rspec spec/conformance_spec.rb .PHONY: test/yugabyte diff --git a/ruby/README.md b/ruby/README.md index af52500cc..32aed0ea2 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -1,6 +1,6 @@ # River for Ruby -River is a fast, reliable background job system backed by Postgres or SQLite. The Ruby client inserts and works jobs using the same database schema and job protocol as River's other languages. It includes Active Record and Sequel drivers, plus Rails and Active Job integration. +River is a fast, reliable background job system backed by MySQL, Postgres, or SQLite. The Ruby client inserts and works jobs using the same database schema and job protocol as River's other languages. It includes Active Record and Sequel drivers, plus Rails and Active Job integration. ## Installation @@ -11,7 +11,7 @@ gem "riverqueue-sequel" gem "pg" ``` -Use `riverqueue-activerecord` for Active Record, or `sqlite3` instead of `pg` for SQLite. Install the gems with `bundle install`. Rails applications can use [`riverqueue-rails`](rails/riverqueue-rails/README.md) to configure River as their Active Job backend. +Use `riverqueue-activerecord` for Active Record, or replace `pg` with `mysql2` for MySQL or `sqlite3` for SQLite. Install the gems with `bundle install`. Rails applications can use [`riverqueue-rails`](rails/riverqueue-rails/README.md) to configure River as their Active Job backend. ## Quick start @@ -67,7 +67,7 @@ ensure end ``` -Run `bundle exec ruby worker.rb`. Replace the worker body with your mail service. Job arguments implement `kind` and `to_json`; workers read decoded JSON keys as strings. Services sharing a job must agree on its kind and JSON fields. For SQLite and alternate Postgres schemas, see [migrations](docs/migrations.md). +Run `bundle exec ruby worker.rb`. Replace the worker body with your mail service. Job arguments implement `kind` and `to_json`; workers read decoded JSON keys as strings. Services sharing a job must agree on its kind and JSON fields. For MySQL, SQLite, and alternate schemas, see [migrations](docs/migrations.md). A client without configured queues can insert jobs without starting worker threads. Applications own their database pools and should close them only after stopping River. @@ -89,3 +89,18 @@ See the [usage and configuration guide](docs/README.md), [migration guide](docs/ ## Development See [developing River for Ruby](docs/development.md). + +## MySQL + +Both Active Record and Sequel drivers support MySQL 8.0 and later. Use the latest +patch release, add `gem "mysql2"` to your bundle, and connect with a +`mysql2://user:password@localhost/my_app` URL. Active Record applications can +configure `adapter: "mysql2"` in their existing database configuration. + +The same insertion, worker, queue, and transaction APIs apply. Workers use +`FOR UPDATE SKIP LOCKED`, and River stores notification hints in the shared +MySQL outbox. Ruby's runtime polls for available work and cancellation requests. +See [migrations](docs/migrations.md) for deployment and MySQL's DDL behavior. + +See the shared [MySQL design notes](../docs/development.md#mysql-design-and-operation) for JSON numeric limits, +exact string comparisons, and server packet-size requirements. diff --git a/ruby/docs/conformance.md b/ruby/docs/conformance.md index c83029282..53a3bec7b 100644 --- a/ruby/docs/conformance.md +++ b/ruby/docs/conformance.md @@ -9,9 +9,16 @@ make test/ruby/conformance ``` This generates `conformance/testdata/` and runs `ruby/spec/conformance_spec.rb`. -No Postgres server, pinned upstream checkout, protocol adapter, or separately -maintained golden files are needed. Missing fixtures fail with instructions to -regenerate them. The regular Ruby suite includes these checks too. +By default, no database server, pinned upstream checkout, protocol adapter, or +separately maintained golden files are needed. Missing fixtures fail with +instructions to regenerate them. The regular Ruby suite includes these checks too. To run the +same persistence cases on MySQL through both Active Record and Sequel: + +```sh +TEST_MYSQL_URL=mysql2://root@localhost/ make test/ruby/conformance/mysql +``` + +Each case creates and drops a disposable database using the canonical migrations. The fixture checks cover: @@ -24,13 +31,13 @@ The fixture checks cover: - Cron occurrences, named zones, daylight-saving transitions, interval schedules, and invalid expressions, with the Ruby API differences below checked explicitly. - Insertion, cancellation, queue pause/resume/metadata, and leadership resignation/request - notification encoding through both SQL drivers, using in-memory SQLite and + notification encoding through both SQL drivers, using SQLite or MySQL and the bundled canonical migrations. The Ruby workflow runs on Ruby changes. The shared Conformance workflow also -runs these tests when Go code changes. The ordinary driver suites retain -Postgres notification delivery/commit-ordering tests and shared Postgres / -SQLite insertion, transaction, uniqueness, and worker tests. +runs these tests on SQLite and MySQL when Go code changes. The ordinary driver +suites retain Postgres notification delivery/commit-ordering tests and shared +Postgres / SQLite / MySQL insertion, transaction, uniqueness, and worker tests. This replaces the old `insert-only-v1` adapter and its pinned external harness. It checks shared data formats, not live mixed-language worker execution. Adapter @@ -49,7 +56,7 @@ remains outside its scope. Shared snooze-counter fixtures cover non-negative integers and absent counters. Recovery from other JSON values is implementation-specific. Ruby's shared driver -tests cover its lenient conversion rules separately on Postgres and SQLite. +tests cover its lenient conversion rules separately on Postgres, SQLite, and MySQL. Cron retains Ruby's existing Fugit extensions: six fields with seconds, Sunday as 7, hour 24, and descending ranges. These four expressions are accepted in diff --git a/ruby/docs/development.md b/ruby/docs/development.md index c0d360e71..c73d9ce42 100644 --- a/ruby/docs/development.md +++ b/ruby/docs/development.md @@ -224,3 +224,9 @@ repository. Follow its README; do not include it in the public release below. ``` 4. Cut a new GitHub release by visiting [new release](https://github.com/riverqueue/river/releases/new), selecting the new tag, and copying in the version's `CHANGELOG.md` content as the release body. + +For MySQL, set `TEST_DATABASE=mysql` and +`TEST_MYSQL_URL=mysql2://root@127.0.0.1/`, then run `make spec/core` from `ruby/`. +The test account must be able to create and drop disposable databases. CI runs +the shared driver, worker, and migration contracts on MySQL 8.0 and 8.4 and +includes their coverage in the combined report. diff --git a/ruby/docs/migrations.md b/ruby/docs/migrations.md index fd9451e4b..a220084fa 100644 --- a/ruby/docs/migrations.md +++ b/ruby/docs/migrations.md @@ -1,7 +1,7 @@ # Schema migrations River Ruby includes a migration API and the `river` command. The gems bundle -byte-for-byte copies of River's canonical Postgres and SQLite migration SQL, +byte-for-byte copies of River's canonical MySQL, Postgres, and SQLite migration SQL, pinned to release v0.48.0 through migration 008; only the upstream schema template placeholders are substituted when executing them. @@ -9,7 +9,7 @@ them. ## Command line Install `riverqueue-sequel` or `riverqueue-activerecord` and the application's -database gem (`pg` or `sqlite3`). The command auto-detects the River driver gem +database gem (`mysql2`, `pg`, or `sqlite3`). The command auto-detects the River driver gem available in your bundle, preferring Sequel when both are available. This choice does not need to match your application's driver: both run the same migrations. Create the database first, then run: @@ -18,6 +18,9 @@ Create the database first, then run: bundle exec river migrate-status --database-url postgres://localhost/my_app bundle exec river migrate-up --database-url postgres://localhost/my_app +# MySQL, using either driver: +bundle exec river migrate-up --database-url mysql2://localhost/my_app + # SQLite, using Sequel: bundle exec river migrate-up --database-url sqlite://storage/river.sqlite3 ``` @@ -114,14 +117,18 @@ migrator.migrate(dry_run: true, target: 8) migrator.migrate(direction: :down) # One version; no interactive confirmation. ``` -The API accepts either driver and optionally `schema:` for Postgres. `migrate` +The API accepts either driver and optionally `schema:` for MySQL or Postgres. `migrate` returns the migration records applied (or planned in dry-run mode), with their -version, name, and original up/down SQL. Each version's SQL and history update +version, name, and original up/down SQL. On Postgres and SQLite, each version's SQL and history update commit in their own transaction. A failing version rolls back while earlier versions remain committed, so rerunning resumes safely. Do not wrap the migrator in an application transaction: some Postgres changes require a real commit before the next version can run. +MySQL DDL commits implicitly. History is recorded after each successful version, +but a failure can leave partially applied DDL. Inspect and repair that version +before retrying. MySQL migrators hold a named lock for the selected database. + An up migration targeting an already-applied version does nothing, even when newer migrations are pending. diff --git a/ruby/driver/riverqueue-activerecord/Gemfile b/ruby/driver/riverqueue-activerecord/Gemfile index c2b4539e3..b7cf311ef 100644 --- a/ruby/driver/riverqueue-activerecord/Gemfile +++ b/ruby/driver/riverqueue-activerecord/Gemfile @@ -11,6 +11,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "debug" gem "pg" gem "rspec-core" diff --git a/ruby/driver/riverqueue-activerecord/Gemfile.lock b/ruby/driver/riverqueue-activerecord/Gemfile.lock index f722d8f72..383fb0364 100644 --- a/ruby/driver/riverqueue-activerecord/Gemfile.lock +++ b/ruby/driver/riverqueue-activerecord/Gemfile.lock @@ -64,6 +64,8 @@ GEM minitest (6.0.6) drb (~> 2.0) prism (~> 1.5) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.28.0) parser (3.3.13.0) @@ -148,6 +150,7 @@ PLATFORMS DEPENDENCIES debug json (< 3) + mysql2 (~> 0.5) pg riverqueue! riverqueue-activerecord! diff --git a/ruby/driver/riverqueue-activerecord/lib/driver.rb b/ruby/driver/riverqueue-activerecord/lib/driver.rb index 2dbfc57bf..8929f8473 100644 --- a/ruby/driver/riverqueue-activerecord/lib/driver.rb +++ b/ruby/driver/riverqueue-activerecord/lib/driver.rb @@ -4,7 +4,7 @@ module River::Driver # Provides an ActiveRecord driver for River supporting Postgres, YugabyteDB, - # and SQLite. + # MySQL, and SQLite. # # Used in conjunction with a River client like: # @@ -17,7 +17,7 @@ class ActiveRecord # Connection class whose pool and transaction context this driver uses. attr_reader :connection_class - # Uses an established Postgres or SQLite connection. The class must be + # Uses an established Postgres, MySQL, or SQLite connection. The class must be # ActiveRecord::Base or an abstract Active Record class. Routing follows its # current role/shard; configure consumers explicitly for each database. def initialize(connection_class: ::ActiveRecord::Base) @@ -32,6 +32,8 @@ def initialize(connection_class: ::ActiveRecord::Base) @connection_class = connection_class @postgres_capabilities_cache = PostgresCapabilities::Cache.new @is_sqlite = connection_class.connection_pool.with_connection { |connection| connection.adapter_name.downcase.include?("sqlite") } + @is_mysql = connection_class.connection_pool.with_connection { |connection| connection.adapter_name.downcase.include?("mysql") } + extend River::Driver::MySQLRuntime if @is_mysql # Do not inherit application scopes, callbacks, or schema caches. Each # driver owns its model, but all operations use the selected class's pool. @job_model = Class.new(::ActiveRecord::Base) do @@ -91,7 +93,11 @@ def rollback_exception end # Backend used by River::Migrator. - def migration_backend = @is_sqlite ? :sqlite : :postgresql + def migration_backend = if @is_mysql + :mysql + else + (@is_sqlite ? :sqlite : :postgresql) + end # Pins a raw connection for migration SQL and refreshes cached model columns. def migration_connection @@ -116,7 +122,9 @@ def transaction(&) # rolling anything back. Let the transaction's owner receive it. next yield if !legacy && connection.current_transaction.joinable? - connection.transaction(requires_new: legacy, &) + options = {requires_new: legacy} + options[:isolation] = :read_committed if @is_mysql && connection.open_transactions.zero? + connection.transaction(**options, &) end end @@ -267,13 +275,15 @@ def transaction(&) end private def runtime_postgres? - !@is_sqlite + !@is_sqlite && !@is_mysql end private def runtime_query_rows(sql) @connection_class.connection_pool.with_connection do |connection| if @is_sqlite connection.raw_connection.execute(sql).to_a + elsif @is_mysql + connection.exec_query(sql).to_a else # exec_query invalidates the query cache for writes with RETURNING. # Deserialize without allocating an Active Record model per job. @@ -292,6 +302,10 @@ def transaction(&) @connection_class.connection_pool.with_connection { |connection| connection.quote(value) } end + private def runtime_unique_violation_class + ::ActiveRecord::RecordNotUnique + end + private def runtime_value(row, key) row[key.to_s] || row[key] end diff --git a/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb b/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb index 124d96abf..ac69fd39b 100644 --- a/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb +++ b/ruby/driver/riverqueue-activerecord/spec/driver_spec.rb @@ -5,6 +5,41 @@ require_relative "../../../spec/row_decoding_shared_examples" RSpec.describe River::Driver::ActiveRecord do + it "selects MySQL through the configured pool and only sets isolation on owned transactions" do + connection = Struct.new(:adapter_name, :open_transactions, :current_transaction, :calls).new( + "Mysql2", 0, Struct.new(:joinable?).new(false), [] + ) + def connection.transaction(**options) + calls << options + yield + end + + def connection.exec_query(sql) + calls << sql + [{"id" => 1}] + end + pool = Object.new + pool.define_singleton_method(:with_connection) { |&block| block.call(connection) } + connection_class = Class.new(ActiveRecord::Base) { self.abstract_class = true } + connection_class.define_singleton_method(:connection_pool) { pool } + driver = described_class.new(connection_class: connection_class) + expect(driver).to be_a(River::Driver::MySQLRuntime) + expect(driver.migration_backend).to eq(:mysql) + expect(driver.send(:runtime_unique_violation_class)).to eq(ActiveRecord::RecordNotUnique) + + expect(driver.transaction { :outer }).to eq(:outer) + connection.open_transactions = 1 + expect(driver.transaction { :nested }).to eq(:nested) + + rows = [{"id" => 1}] + expect(driver.send(:runtime_query_rows, "SELECT id FROM river_job")).to eq(rows) + expect(connection.calls).to eq([ + {requires_new: false, isolation: :read_committed}, + {requires_new: false}, + "SELECT id FROM river_job" + ]) + end + before do if ENV["RIVER_DEBUG"] == "1" || ENV["RIVER_DEBUG"] == "true" ActiveRecord::Base.logger = Logger.new($stdout) diff --git a/ruby/driver/riverqueue-sequel/Gemfile b/ruby/driver/riverqueue-sequel/Gemfile index dd40424c6..1aaa25252 100644 --- a/ruby/driver/riverqueue-sequel/Gemfile +++ b/ruby/driver/riverqueue-sequel/Gemfile @@ -11,6 +11,7 @@ group :development, :test do end group :test do + gem "mysql2", "~> 0.5" gem "pg" gem "rspec-core" gem "rspec-expectations" diff --git a/ruby/driver/riverqueue-sequel/Gemfile.lock b/ruby/driver/riverqueue-sequel/Gemfile.lock index 2a9132a9a..cd57075ce 100644 --- a/ruby/driver/riverqueue-sequel/Gemfile.lock +++ b/ruby/driver/riverqueue-sequel/Gemfile.lock @@ -24,6 +24,8 @@ GEM language_server-protocol (3.17.0.6) lint_roller (1.1.0) logger (1.7.0) + mysql2 (0.5.7) + bigdecimal optparse (0.8.1) parallel (1.28.0) parser (3.3.13.0) @@ -92,6 +94,7 @@ PLATFORMS DEPENDENCIES json (< 3) + mysql2 (~> 0.5) pg riverqueue! riverqueue-sequel! diff --git a/ruby/driver/riverqueue-sequel/lib/driver.rb b/ruby/driver/riverqueue-sequel/lib/driver.rb index 69a69bfb8..655b78340 100644 --- a/ruby/driver/riverqueue-sequel/lib/driver.rb +++ b/ruby/driver/riverqueue-sequel/lib/driver.rb @@ -3,7 +3,7 @@ require "securerandom" module River::Driver - # Provides a Sequel driver for River supporting Postgres, YugabyteDB, and SQLite. + # Provides a Sequel driver for River supporting Postgres, YugabyteDB, MySQL, and SQLite. # # Used in conjunction with a River client like: # @@ -18,14 +18,16 @@ module River::Driver class Sequel include River::Driver::Runtime - # Creates a driver backed by a connected Sequel::Database for Postgres or + # Creates a driver backed by a connected Sequel::Database for Postgres, MySQL, or # SQLite. def initialize(db) @db = db @postgres_capabilities_cache = PostgresCapabilities::Cache.new @is_sqlite = (db.database_type == :sqlite) + @is_mysql = (db.database_type == :mysql) + extend River::Driver::MySQLRuntime if @is_mysql - unless @is_sqlite + unless @is_sqlite || @is_mysql db.extension(:pg_array) db.extension(:pg_json) end @@ -80,7 +82,11 @@ def rollback_exception end # Backend used by River::Migrator. - def migration_backend = @is_sqlite ? :sqlite : :postgresql + def migration_backend = if @is_mysql + :mysql + else + (@is_sqlite ? :sqlite : :postgresql) + end # Pins a raw connection for the complete migration operation. def migration_connection @@ -97,6 +103,7 @@ def migration_connection def transaction block_error = nil options = runtime_legacy_subtransactions? ? {savepoint: true} : {} + options[:isolation] = :committed if @is_mysql && !@db.in_transaction? @db.transaction(**options) do yield rescue ArgumentError => error @@ -237,7 +244,7 @@ def transaction end private def runtime_postgres? - !@is_sqlite + !@is_sqlite && !@is_mysql end private def runtime_query_rows(sql) @@ -248,6 +255,10 @@ def transaction @db.literal(value) end + private def runtime_unique_violation_class + ::Sequel::UniqueConstraintViolation + end + private def runtime_value(row, key) row[key] || row[key.to_s] end diff --git a/ruby/driver/riverqueue-sequel/spec/driver_spec.rb b/ruby/driver/riverqueue-sequel/spec/driver_spec.rb index ce1b4ce82..c6eb0f4d3 100644 --- a/ruby/driver/riverqueue-sequel/spec/driver_spec.rb +++ b/ruby/driver/riverqueue-sequel/spec/driver_spec.rb @@ -5,6 +5,23 @@ require_relative "../../../spec/row_decoding_shared_examples" RSpec.describe River::Driver::Sequel do + it "selects the MySQL runtime and migrations for mysql2 connections" do + database = Sequel.mock + def database.database_type = :mysql + driver = described_class.new(database) + expect(driver).to be_a(River::Driver::MySQLRuntime) + expect(driver.migration_backend).to eq(:mysql) + expect(driver.send(:runtime_unique_violation_class)).to eq(Sequel::UniqueConstraintViolation) + calls = [] + original = database.method(:transaction) + database.define_singleton_method(:transaction) do |**options, &block| + calls << options + original.call(**options, &block) + end + expect(driver.transaction { driver.transaction { :nested } }).to eq(:nested) + expect(calls).to eq([{isolation: :committed}, {}]) + end + if DB context "with Postgres", database: :postgres do around(:each) { |ex| test_transaction(&ex) } diff --git a/ruby/lib/driver.rb b/ruby/lib/driver.rb index d88368639..c701708e6 100644 --- a/ruby/lib/driver.rb +++ b/ruby/lib/driver.rb @@ -52,3 +52,4 @@ def initialize( require_relative "driver/job_row_decoder" require_relative "driver/notification_listener" require_relative "driver/runtime" +require_relative "driver/mysql_runtime" diff --git a/ruby/lib/driver/mysql_runtime.rb b/ruby/lib/driver/mysql_runtime.rb new file mode 100644 index 000000000..1c33af52e --- /dev/null +++ b/ruby/lib/driver/mysql_runtime.rb @@ -0,0 +1,413 @@ +# frozen_string_literal: true + +module River::Driver + # MySQL statements shared by the Active Record and Sequel adapters. Writes + # lock and read back their rows in the same transaction because MySQL has no + # RETURNING clause. Only MySQL driver instances extend this module. + module MySQLRuntime + def job_cancel(id, now: Time.now.utc) + transaction do + rows = mysql_update("river_job", "id", "id = #{Integer(id)} AND state NOT IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NULL", [ + "finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE #{runtime_time(now)} END", + "metadata = #{runtime_merge_metadata("cancel_attempted_at" => now.iso8601(6))}", + "state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END" + ]) + job = job_get_by_id(id) + runtime_notify("river_control", action: "cancel", job_id: job.id, queue: job.queue) unless rows.empty? + job + end + end + + def job_complete(id:, finalized_at:, expected_job: nil, metadata: nil, now: Time.now.utc) + assignments = ["finalized_at = #{runtime_time(finalized_at)}", "state = 'completed'"] + assignments << "metadata = #{runtime_merge_metadata(metadata)}" if metadata && !metadata.empty? + transaction do + rows = mysql_update("river_job", "id", runtime_running_job_predicate(id, expected_job: expected_job, uncancelled: true), assignments) + next mysql_runtime_job(rows.first) unless rows.empty? + + predicate = "id = #{Integer(id)} AND (#{runtime_cancel_attempted})" + predicate += " AND #{runtime_job_attempt_predicate(expected_job)}" if expected_job + :cancelled if runtime_query_rows("SELECT id FROM river_job WHERE #{predicate}").any? + end + end + + def job_delete(id) + transaction do + rows = mysql_delete("river_job", "id", "id = #{Integer(id)} AND state != 'running'") + rows.empty? ? job_get_by_id(id) : runtime_decode_sqlite_job(rows.first) + end + end + + def job_delete_finalized(retention:, now: Time.now.utc, max: 1_000) + clauses = retention.filter_map do |state, seconds| + "(state = #{runtime_quote(state.to_s)} AND finalized_at < #{runtime_time(now - seconds)})" if seconds + end + return 0 if clauses.empty? + + transaction do + predicate = "(#{clauses.join(" OR ")})" + ids = runtime_query_rows("SELECT id FROM river_job WHERE #{predicate} ORDER BY id LIMIT #{Integer(max)} FOR UPDATE SKIP LOCKED") + .map { |row| runtime_value(row, :id) } + next 0 if ids.empty? + + mysql_delete("river_job", "id", "#{runtime_in_clause("id", ids)} AND #{predicate}").length + end + end + + def job_delete_if_running(id) + transaction do + deleted = mysql_delete("river_job", "id", "id = #{Integer(id)} AND state = 'running' AND NOT (#{runtime_cancel_attempted})") + next true unless deleted.empty? + + job_get_cancelled_ids([Integer(id)]).include?(Integer(id)) ? :cancelled : false + end + end + + def job_delete_many(params) + transaction do + jobs = runtime_job_list(params, for_delete: true) + next [] if jobs.empty? + + mysql_delete("river_job", "id", runtime_in_clause("id", jobs.map(&:id)) + " AND state != 'running'") + jobs + end + end + + def job_get_by_id(id) + row = runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job WHERE id = #{Integer(id)}").first + row ? runtime_decode_sqlite_job(row) : nil + end + + def job_insert_many(params) + return [] if params.empty? + + nonces = sqlite_insert_nonces(params) + transaction do + results = params.zip(nonces).map do |param, nonce| + metadata = (param.metadata || {}).merge("river:unique_nonce" => nonce) + values = [runtime_json(param.encoded_args), runtime_quote(param.kind), Integer(param.max_attempts), + runtime_json(metadata), Integer(param.priority), runtime_quote(param.queue), + param.scheduled_at ? runtime_time(param.scheduled_at) : "UTC_TIMESTAMP(6)", + runtime_quote(param.state), runtime_json(param.tags || []), + param.unique_key ? "UNHEX(#{runtime_quote(param.unique_key.unpack1("H*"))})" : "NULL", + param.unique_states ? param.unique_states.to_i(2) : "NULL"] + runtime_execute(<<~SQL) + INSERT INTO river_job (args, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) + VALUES (#{values.join(",")}) + ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id) + SQL + id = runtime_value(runtime_query_rows("SELECT LAST_INSERT_ID() AS id").first, :id) + job = job_get_by_id(id) + duplicate = job.metadata["river:unique_nonce"] != nonce + if duplicate && (param.unique_key.nil? || job.unique_key != param.unique_key || + !job.unique_states&.include?(job.state) || + !River::UniqueBitmask.to_states((param.unique_states || "0").to_i(2)).include?(param.state)) + raise River::Error, "insert conflicts with a constraint other than River's unique-job index" + end + [job, duplicate] + end + params.select { |param| param.state == River::JOB_STATE_AVAILABLE }.map(&:queue).uniq.each do |queue| + runtime_notify("river_insert", queue: queue) + end + results + end + end + + def job_metadata_merge(id, metadata, expected_job = nil) + predicate = "id = #{Integer(id)}" + if expected_job + predicate += " AND #{runtime_job_attempt_predicate(expected_job)}" + predicate += " AND state = #{runtime_quote(expected_job.state)}" + end + rows = mysql_update("river_job", "id", predicate, ["metadata = #{runtime_merge_metadata(metadata)}"]) + rows.empty? ? nil : runtime_decode_sqlite_job(rows.first) + end + + def job_retry(id, now: Time.now.utc) + transaction do + job = mysql_locked_job(id) + next job if !job || job.state == "running" || (job.state == "available" && job.scheduled_at <= now) + raise ArgumentError, "cannot retry a job with #{River::MAX_ATTEMPTS_LIMIT} or more attempts" if job.attempt >= River::MAX_ATTEMPTS_LIMIT + + mysql_update("river_job", "id", "id = #{Integer(id)}", [ + "max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END", + "metadata = JSON_REMOVE(metadata, '$.cancel_attempted_at')", "finalized_at = NULL", + "scheduled_at = #{runtime_time(now)}", "state = 'available'" + ]) + runtime_notify("river_insert", queue: job.queue) + job_get_by_id(id) + end + end + + def job_schedule(now: Time.now.utc, max: 1_000) + transaction do + rows = runtime_query_rows(<<~SQL) + SELECT id, queue FROM river_job + WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= #{runtime_time(now)} + ORDER BY priority, scheduled_at, id LIMIT #{Integer(max)} FOR UPDATE SKIP LOCKED + SQL + rows.each do |row| + id = runtime_value(row, :id).to_i + begin + runtime_execute("UPDATE river_job SET state = 'available' WHERE id = #{id}") + rescue runtime_unique_violation_class + runtime_execute(<<~SQL) + UPDATE river_job SET state = 'discarded', finalized_at = #{runtime_time(now)}, + metadata = #{runtime_merge_metadata("unique_key_conflict" => "scheduler_discarded")} + WHERE id = #{id} + SQL + end + end + rows.map { |row| runtime_value(row, :queue) }.uniq.each { |queue| runtime_notify("river_insert", queue: queue) } + rows.length + end + end + + def job_set_state_if_running(id:, state:, now: Time.now.utc, attempt: nil, + error: nil, finalized_at: nil, metadata: nil, scheduled_at: nil) + cancel = runtime_cancel_attempted + assignments = [] #: Array[String] + assignments << "attempt = CASE WHEN NOT (#{cancel}) THEN #{Integer(attempt)} ELSE attempt END" unless attempt.nil? + assignments << "errors = #{runtime_append_error(error)}" if error + assignments << "finalized_at = CASE WHEN #{cancel} THEN #{runtime_time(now)} ELSE #{runtime_nullable_time(finalized_at)} END" + assignments << "scheduled_at = CASE WHEN NOT (#{cancel}) THEN #{runtime_time(scheduled_at)} ELSE scheduled_at END" if scheduled_at + # MySQL evaluates assignments left-to-right. Check cancellation before + # replacing metadata or state. + assignments << "state = CASE WHEN #{cancel} THEN 'cancelled' ELSE #{runtime_state(state.to_s)} END" + assignments << "metadata = #{runtime_merge_metadata(metadata)}" if metadata && !metadata.empty? + rows = mysql_update("river_job", "id", "id = #{Integer(id)} AND state = 'running'", assignments) + rows.empty? ? nil : mysql_runtime_job(rows.first) + end + + def job_update(id, params) + values = params.each.to_h + assignments = [] #: Array[String] + if values.key?(:output) + output = values.delete(:output) + if values.key?(:metadata) + metadata = values[:metadata] + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + values[:metadata] = metadata.transform_keys(&:to_s).merge("output" => output) + else + # Merge against the database row in the UPDATE, preserving metadata + # written since the caller last read the job. + assignments << "metadata = #{runtime_merge_metadata("output" => output)}" + end + end + values.each do |field, value| + raise ArgumentError, "unknown update field: #{field}" unless River::JobUpdateParams.method_defined?(field) + + assignments << "#{field} = #{runtime_update_value(field, value)}" + end + + return job_get_by_id(id) if assignments.empty? + + rows = mysql_update("river_job", "id", "id = #{Integer(id)}", assignments) + rows.empty? ? nil : runtime_decode_sqlite_job(rows.first) + end + + def leader_acquire(id, ttl: 30, now: Time.now.utc) + transaction do + runtime_execute("DELETE FROM river_leader WHERE expires_at < #{runtime_time(now)}") + runtime_execute(<<~SQL) + INSERT INTO river_leader (leader_id, elected_at, expires_at) + VALUES (#{runtime_quote(id)}, #{runtime_time(now)}, #{runtime_time(now + ttl)}) + ON DUPLICATE KEY UPDATE name = name + SQL + runtime_query_rows("SELECT leader_id FROM river_leader WHERE leader_id = CAST(#{runtime_quote(id)} AS BINARY) FOR UPDATE").any? + end + end + + def leader_release(id) + transaction do + deleted = mysql_delete("river_leader", "name", "leader_id = CAST(#{runtime_quote(id)} AS BINARY)") + runtime_notify("river_leadership", action: "resigned", leader_id: id) unless deleted.empty? + end + end + + def leader_renew(id, ttl: 30, now: Time.now.utc) + mysql_update("river_leader", "name", "leader_id = CAST(#{runtime_quote(id)} AS BINARY) AND expires_at >= #{runtime_time(now)}", + ["expires_at = #{runtime_time(now + ttl)}"]).any? + end + + def normalize_time(time) + time.getutc.floor(6) + end + + def notification_delete_before(horizon:, max: 10_000) + mysql_delete("river_notification", "id", "created_at < #{runtime_time(horizon)}", limit: max).length + end + + def notification_listener + NotificationListener::MySQL.new(->(sql) { transaction { runtime_query_rows(sql) } }) + end + + def queue_pause(name, now: Time.now.utc) + mysql_queue_paused(name, true, now) + end + + def queue_resume(name, now: Time.now.utc) + mysql_queue_paused(name, false, now) + end + + def queue_update(name, metadata:, now: Time.now.utc) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + transaction do + rows = mysql_update("river_queue", "name", "name = CAST(#{runtime_quote(name)} AS BINARY)", + ["metadata = #{runtime_json(metadata)}", "updated_at = #{runtime_time(now)}"]) + next nil if rows.empty? + + runtime_notify("river_control", action: "metadata_changed", queue: name, metadata: metadata) + runtime_queue_from_row(rows.first) + end + end + + def queue_upsert(name, metadata: {}, now: Time.now.utc) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + transaction do + runtime_execute(<<~SQL) + INSERT INTO river_queue (name, created_at, metadata, updated_at) + VALUES (#{runtime_quote(name)}, #{runtime_time(now)}, #{runtime_json(metadata)}, #{runtime_time(now)}) + ON DUPLICATE KEY UPDATE updated_at = #{runtime_time(now)} + SQL + queue_get(name) + end + end + + def time_precision + Rational(1, 1_000_000) + end + + private def mysql_columns(table) + columns = case table + when "river_job" + %w[id args attempt attempted_at attempted_by created_at errors finalized_at kind max_attempts metadata priority queue scheduled_at state tags unique_key unique_states] + when "river_queue" + %w[name created_at metadata paused_at updated_at] + else + return "*" + end + columns.map { |column| column.end_with?("_at") ? "CAST(#{column} AS CHAR) AS #{column}" : column }.join(", ") + end + + private def mysql_delete(table, key, predicate, limit: nil) + transaction do + hint = (table == "river_job") ? "/*+ JOIN_INDEX(river_job PRIMARY) */ " : "" + rows = runtime_query_rows("SELECT #{hint}#{mysql_columns(table)} FROM #{table} WHERE #{predicate} ORDER BY #{key} #{"LIMIT #{Integer(limit)}" if limit} FOR UPDATE") + runtime_execute("DELETE /*+ JOIN_INDEX(#{table} PRIMARY) */ FROM #{table} WHERE #{runtime_in_clause(key, rows.map { |row| runtime_value(row, key.to_sym) })}") unless rows.empty? + rows + end + end + + private def mysql_locked_job(id) + row = runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job WHERE id = #{Integer(id)} FOR UPDATE").first + row ? runtime_decode_sqlite_job(row) : nil + end + + private def mysql_queue_paused(name, paused, now) + transaction do + predicate = (name == "*") ? "true" : "name = CAST(#{runtime_quote(name)} AS BINARY)" + assignments = ["updated_at = CASE WHEN paused_at IS #{paused ? "NULL" : "NOT NULL"} THEN #{runtime_time(now)} ELSE updated_at END", + "paused_at = #{paused ? "COALESCE(paused_at, #{runtime_time(now)})" : "NULL"}"] + rows = mysql_update("river_queue", "name", predicate, assignments) + runtime_notify("river_control", action: paused ? "pause" : "resume", queue: name) unless rows.empty? + rows.map { |row| runtime_queue_from_row(row) } + end + end + + private def mysql_runtime_job(row) + runtime_decode_sqlite_job(row) + rescue River::JobRowDecodeError => error + error.job + end + + private def mysql_update(table, key, predicate, assignments) + transaction do + rows = runtime_query_rows("SELECT #{key} FROM #{table} WHERE #{predicate} ORDER BY #{key} FOR UPDATE") + next [] if rows.empty? + + filter = runtime_in_clause(key, rows.map { |row| runtime_value(row, key.to_sym) }) + runtime_execute("UPDATE #{table} SET #{assignments.join(", ")} WHERE #{filter}") + runtime_query_rows("SELECT #{mysql_columns(table)} FROM #{table} WHERE #{filter} ORDER BY #{key}") + end + end + + private def runtime_append_error(error) + value = error.respond_to?(:to_h) ? error.to_h : error + "JSON_ARRAY_APPEND(CASE WHEN JSON_TYPE(errors) = 'ARRAY' THEN errors WHEN errors IS NULL THEN JSON_ARRAY() ELSE JSON_ARRAY(errors) END, '$', #{runtime_json(value)})" + end + + private def runtime_cancel_attempted = "JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at')" + + private def runtime_claim_jobs(predicate, max, attempted_by, now) + transaction do + ids = runtime_query_rows("SELECT id FROM river_job WHERE #{predicate} ORDER BY priority, scheduled_at, id LIMIT #{Integer(max)} FOR UPDATE SKIP LOCKED") + .map { |row| runtime_value(row, :id) } + next [] if ids.empty? + + runtime_execute(<<~SQL) + UPDATE river_job SET attempt = LEAST(attempt + 1, 32767), + attempted_at = #{runtime_time(now)}, + attempted_by = JSON_ARRAY_APPEND(CASE WHEN JSON_LENGTH(attempted_by) >= 100 THEN JSON_EXTRACT(attempted_by, '$[last-98 to last]') ELSE COALESCE(attempted_by, JSON_ARRAY()) END, '$', #{runtime_quote(attempted_by)}), + state = 'running' + WHERE #{runtime_in_clause("id", ids)} + SQL + ids.map { |id| runtime_read_job(id) } + end + end + + private def runtime_in_clause(column, values) + predicate = super + return predicate unless %w[kind name queue].include?(column) + + # Keep the indexed comparison so claims do not scan and lock other queues. + "(#{predicate} AND #{super("CAST(#{column} AS BINARY)", values)})" + end + + private def runtime_job_attempt_predicate(job) + "attempt = #{Integer(job.attempt)} AND attempted_at <=> #{runtime_nullable_time(job.attempted_at)}" + end + + private def runtime_job_rows(suffix) + runtime_query_rows("SELECT #{mysql_columns("river_job")} FROM river_job #{suffix}").map { |row| runtime_decode_sqlite_job(row) } + end + + private def runtime_json(value) + encoded = value.is_a?(String) ? value : JSON.generate(value) + "CAST(#{runtime_quote(encoded)} AS JSON)" + end + + private def runtime_locks_rows? = true + + private def runtime_merge_metadata(metadata) + raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash) + + metadata.reduce("metadata") do |expression, (key, value)| + "JSON_SET(#{expression}, #{runtime_quote("$.#{JSON.generate(key.to_s)}")}, CAST(#{runtime_quote(JSON.generate(value))} AS JSON))" + end + end + + private def runtime_metadata_equals(key, value) + "JSON_EXTRACT(metadata, #{runtime_quote("$.#{JSON.generate(key.to_s)}")}) = CAST(#{runtime_quote(JSON.generate(value))} AS JSON)" + end + + private def runtime_order_by(params, null_order) + "#{"#{params.sort_by} IS NULL, " unless null_order.empty?}#{params.sort_by} #{params.sort_order.to_s.upcase}, id #{params.sort_order.to_s.upcase}" + end + + private def runtime_queue_columns = mysql_columns("river_queue") + + private def runtime_tag_contains(tag) + "JSON_CONTAINS(tags, JSON_QUOTE(#{runtime_quote(tag)}))" + end + + private def runtime_time(value) + raise ArgumentError, "time cannot be nil" unless value + + runtime_quote(value.getutc.strftime("%Y-%m-%d %H:%M:%S.%6N")) + end + end +end diff --git a/ruby/lib/driver/notification_listener.rb b/ruby/lib/driver/notification_listener.rb index dd6d2c12c..22dcad973 100644 --- a/ruby/lib/driver/notification_listener.rb +++ b/ruby/lib/driver/notification_listener.rb @@ -37,14 +37,14 @@ class SQLite def initialize(query) @query = query # Historical requests must not resign a newly started client. - @last_id = @query.call("SELECT coalesce(max(id), 0) AS id FROM river_notification").first.transform_keys(&:to_sym).fetch(:id).to_i + @last_id = initial_cursor end def close end def poll(timeout) - rows = @query.call("SELECT id, topic, payload FROM river_notification WHERE id > #{@last_id} ORDER BY id LIMIT 1000") + rows = read_batch if rows.empty? sleep(timeout) return [] @@ -56,6 +56,27 @@ def poll(timeout) [row.fetch(:topic), row.fetch(:payload)] end end + + private def initial_cursor + @query.call("SELECT coalesce(max(id), 0) AS id FROM river_notification").first.transform_keys(&:to_sym).fetch(:id).to_i + end + + private def read_batch + @query.call("SELECT id, topic, payload FROM river_notification WHERE id > #{@last_id} ORDER BY id LIMIT 1000") + end + end + + class MySQL < SQLite + # Locking scans wait for uncommitted lower IDs before advancing the cursor. + # A snapshot MAX(id) could otherwise permanently skip a later commit. + private def initial_cursor + row = @query.call("SELECT id FROM river_notification ORDER BY id FOR UPDATE").last + row ? row.transform_keys(&:to_sym).fetch(:id).to_i : 0 + end + + private def read_batch + @query.call("SELECT id, topic, payload FROM river_notification WHERE id > #{@last_id} ORDER BY id LIMIT 1000 FOR SHARE") + end end end end diff --git a/ruby/lib/driver/runtime.rb b/ruby/lib/driver/runtime.rb index 87bf3223e..7033c427e 100644 --- a/ruby/lib/driver/runtime.rb +++ b/ruby/lib/driver/runtime.rb @@ -148,7 +148,7 @@ def job_delete_many(params) def job_get_available(queue:, max:, attempted_by:, kinds: nil, now: Time.now.utc) return [] if kinds&.empty? - predicate = "state = 'available' AND queue = #{runtime_quote(queue)} AND scheduled_at <= #{runtime_time(now)}" + predicate = "state = 'available' AND #{runtime_in_clause("queue", [queue])} AND scheduled_at <= #{runtime_time(now)}" predicate += " AND #{runtime_in_clause("kind", kinds)}" if kinds runtime_claim_jobs(predicate, max, attempted_by, now) end @@ -190,7 +190,7 @@ def job_rescue_stuck(horizon:, retry_policy:, now: Time.now.utc, max: 1_000, log # Lock each candidate until its transition finishes. Continue past jobs # that are still within their timeouts without consuming the rescue limit. after_id = rescued = 0 - lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : "" + lock = runtime_locks_rows? ? "FOR UPDATE SKIP LOCKED" : "" while rescued < max ids = runtime_returning_ids(<<~SQL) SELECT id FROM river_job @@ -393,7 +393,7 @@ def notification_listener end def queue_get(name) - row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE name = #{runtime_quote(name)}").first + row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE #{runtime_in_clause("name", [name])}").first runtime_queue_from_row(row) end @@ -659,9 +659,9 @@ def time_precision where = clauses.empty? ? "" : "WHERE #{clauses.join(" AND ")}" runtime_job_rows(<<~SQL) #{where} - ORDER BY #{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase} + ORDER BY #{runtime_order_by(params, null_order)} LIMIT #{params.limit} - #{"FOR UPDATE SKIP LOCKED" if for_delete && runtime_postgres?} + #{"FOR UPDATE SKIP LOCKED" if for_delete && runtime_locks_rows?} SQL end @@ -672,6 +672,12 @@ def time_precision runtime_postgres? ? "state::text" : "state" end + private def runtime_locks_rows? = runtime_postgres? + + private def runtime_order_by(params, null_order) + "#{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase}" + end + private def runtime_json(value) encoded = value.is_a?(String) ? value : JSON.generate(value) runtime_postgres? ? "#{runtime_quote(encoded)}::jsonb" : "jsonb(#{runtime_quote(encoded)})" diff --git a/ruby/lib/migration_cli.rb b/ruby/lib/migration_cli.rb index 5cfd2f962..ee25bada2 100644 --- a/ruby/lib/migration_cli.rb +++ b/ruby/lib/migration_cli.rb @@ -114,9 +114,10 @@ def self.export(options, migrator_class, out:) raise ArgumentError, "pass exactly one of --up or --down" if !!options[:up] == !!options[:down] backend = case options[:url] || "postgres://" + when /\Amysql2?:/i then :mysql when /\Apostgres(?:ql)?:/i then :postgresql when /\Asqlite3?:/i then :sqlite - else raise ArgumentError, "--database-url must use postgres:, postgresql:, sqlite:, or sqlite3:" + else raise ArgumentError, "--database-url must use mysql:, mysql2:, postgres:, postgresql:, sqlite:, or sqlite3:" end schema = options[:schema] raise ArgumentError, "SQLite does not support a migration schema option" if backend == :sqlite && schema @@ -136,7 +137,12 @@ def self.export(options, migrator_class, out:) out.puts if index.positive? out.puts("-- River #{options[:line]} migration #{migration.version.to_s.rjust(3, "0")} [#{direction}]") sql = options[:down] ? migration.sql_down : migration.sql_up - out.puts(sql.gsub("/* TEMPLATE: schema */", schema ? %("#{schema}".) : "").strip) + schema_prefix = if schema + (backend == :mysql) ? "`#{schema}`." : %("#{schema}".) + else + "" + end + out.puts(sql.gsub("/* TEMPLATE: schema */", schema_prefix).strip) end 0 end diff --git a/ruby/lib/migrator.rb b/ruby/lib/migrator.rb index fe5b89f1f..39cca421c 100644 --- a/ruby/lib/migrator.rb +++ b/ruby/lib/migrator.rb @@ -25,7 +25,7 @@ def initialize(driver, line: "main", migrations_path: File.expand_path("../migra # Returns bundled SQL without a driver or database connection. def self.migrations(backend: :postgresql, line: "main", migrations_path: File.expand_path("../migration", __dir__.to_s)) - raise ArgumentError, "unsupported migration backend" unless [:postgresql, :sqlite].include?(backend) + raise ArgumentError, "unsupported migration backend" unless [:mysql, :postgresql, :sqlite].include?(backend) raise ArgumentError, "invalid migration line" unless line.match?(/\A[a-z][a-z0-9_]*\z/) directory = File.join(migrations_path, backend.to_s, line) @@ -68,7 +68,7 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) unless dry_run plan.each do |migration| - execute((@backend == :sqlite) ? "BEGIN IMMEDIATE" : "BEGIN") + execute((@backend == :sqlite) ? "BEGIN IMMEDIATE" : "BEGIN") unless @backend == :mysql committed = false begin raise River::Error, "migration state changed concurrently; rerun migration" unless existing_versions == existing @@ -78,7 +78,7 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) end sql = (direction == :up) ? migration.sql_up : migration.sql_down - execute(sql.gsub("/* TEMPLATE: schema */", @schema ? %("#{@schema}".) : "")) + execute(sql.gsub("/* TEMPLATE: schema */", schema_prefix)) if direction == :up columns, values = line_column? ? ["line, version", "'#{@line}', #{migration.version}"] : ["version", migration.version.to_s] @@ -93,10 +93,10 @@ def migrate(direction: :up, dry_run: false, steps: nil, target: nil) existing -= [migration.version] end - execute("COMMIT") + execute("COMMIT") unless @backend == :mysql committed = true ensure - execute("ROLLBACK") unless committed + execute("ROLLBACK") unless committed || @backend == :mysql end end end @@ -116,6 +116,10 @@ def status private def execute(sql) if @backend == :postgresql @connection.exec(sql) + elsif @backend == :mysql + # MySQL DDL commits implicitly. Run statements separately without + # requiring MULTI_STATEMENTS on the application's connection pool. + mysql_statements(sql).each { |statement| @connection.query(statement) } else @connection.execute_batch(sql) end @@ -124,6 +128,8 @@ def status private def existing_versions exists = if @backend == :postgresql query("SELECT to_regclass('#{table}') AS name").first.fetch("name") + elsif @backend == :mysql + query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{@schema}' AND table_name = 'river_migration'").any? else query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'river_migration'").any? end @@ -145,7 +151,7 @@ def status end private def line_column? - if @backend == :postgresql + if [:mysql, :postgresql].include?(@backend) query("SELECT column_name FROM information_schema.columns WHERE table_schema = '#{@schema}' AND table_name = 'river_migration' AND column_name = 'line'").any? else query("PRAGMA table_info(river_migration)").any? { |row| row.fetch("name") == "line" } @@ -155,6 +161,8 @@ def status private def query(sql) if @backend == :postgresql @connection.exec(sql).to_a + elsif @backend == :mysql + @connection.query(sql, as: :hash, symbolize_keys: false).to_a else rows = @connection.execute2(sql) columns = rows.shift @@ -168,6 +176,16 @@ def status @connection = connection @schema = if @backend == :postgresql @requested_schema || query("SELECT current_schema() AS name").first.fetch("name") + elsif @backend == :mysql + @requested_schema || query("SELECT DATABASE() AS name").first.fetch("name") + end + if @backend == :mysql + raise ArgumentError, "schema must be a simple SQL identifier" unless @schema&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]{0,63}\z/) + + if lock + locked = query("SELECT GET_LOCK(CONCAT('river_migrate:',LEFT(SHA2('#{@schema}',256),48)), 0) AS locked").first.fetch("locked") + raise River::Error, "another migrator holds the database lock" unless locked == 1 + end end if @backend == :postgresql raise ArgumentError, "schema must be a simple SQL identifier" unless @schema&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/) @@ -183,6 +201,8 @@ def status ensure if @backend == :postgresql && lock query("SELECT pg_advisory_unlock(hashtext(current_database()), hashtext('river_migrate:#{@schema}'))") + elsif @backend == :mysql && lock + query("SELECT RELEASE_LOCK(CONCAT('river_migrate:',LEFT(SHA2('#{@schema}',256),48)))") end end end @@ -190,7 +210,33 @@ def status end private def table - @schema ? %("#{@schema}".river_migration) : "river_migration" + "#{schema_prefix}river_migration" + end + + private def schema_prefix + return "" unless @schema + + (@backend == :mysql) ? "`#{@schema}`." : %("#{@schema}".) + end + + private def mysql_statements(sql) + statements = [] #: Array[String] + buffer = +"" + # Quoted text and comments are tokens so their semicolons cannot split + # statements. River migrations contain plain SQL, not DELIMITER commands. + sql.scan(/'(?:''|\\.|[^'\\])*'|"(?:""|\\.|[^"\\])*"|`(?:``|[^`])*`|--[^\n]*|\#[^\n]*|\/\*.*?\*\/|;|./m) do |token| + token = token.to_s + if token.start_with?("--", "#", "/*") + buffer << " " + elsif token == ";" + statements << buffer.strip unless buffer.strip.empty? + buffer = +"" + else + buffer << token + end + end + statements << buffer.strip unless buffer.strip.empty? + statements end end end diff --git a/ruby/migration/manifest.json b/ruby/migration/manifest.json index 100357042..b4e1e5567 100644 --- a/ruby/migration/manifest.json +++ b/ruby/migration/manifest.json @@ -1,5 +1,21 @@ { "files": { + "mysql/main/001_create_river_migration.down.sql": "660d6b11eeb88589d3ce2053be165cda8df4c1d0172fbc1e8c3daa614e3f5b93", + "mysql/main/001_create_river_migration.up.sql": "b0d3147cf330526af7f3478b336f3f9f929bb9f217db08da12ca821e6d5568fd", + "mysql/main/002_initial_schema.down.sql": "93d74166cb838b13ae42447181afb2c2987dcb16ae33422de612daecb0163e79", + "mysql/main/002_initial_schema.up.sql": "2c5ab6645cf016155ad7fd5c46929ddc8bf76006cb6e7b85601bdadcf945f504", + "mysql/main/003_river_job_tags_non_null.down.sql": "dbfb0192d6254bdcf2a12c70c8b4ad44dfd7ebaa273b3160c0e76b9208f0a1fe", + "mysql/main/003_river_job_tags_non_null.up.sql": "e8a8459edd8e6110e564be74c03f646b430e2f3e31fef31eeb5610ed3b10660f", + "mysql/main/004_pending_and_more.down.sql": "f4325dd580c3725d7eb068262d56319f779b556ceddfb53771c51a855ca2440f", + "mysql/main/004_pending_and_more.up.sql": "c4642feef2bddc16ca0c5dcdf84942abb219ed40b1427b571673e8f5360bbb53", + "mysql/main/005_migration_unique_client.down.sql": "bd06f54a7fd083e4962355d67186fcd09f0f8de6392d18f406003fba0732018f", + "mysql/main/005_migration_unique_client.up.sql": "09e8dff3080a05bfc10ff260bb96508cb0e9639a3f9dc86505c50ef6f931dcc3", + "mysql/main/006_bulk_unique.down.sql": "5c86b3bd5e32a1f8462b6e4f7ba8aa4fcb4c5be37f4c7d3c1faeb9602ca410fe", + "mysql/main/006_bulk_unique.up.sql": "f8625bc2fbff65b1b77fb416bc95889981256d890ce8d157bb1227064bff5228", + "mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql": "d25dbb01d5c4385f40581fe9eb54f26aa71709cb7f10050803c5e1fc7a8bf46c", + "mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql": "25cc59f31682454bc497962a3a98cf29f5760e3341e2bfa37fcef108a86d9b96", + "mysql/main/008_job_id_autoincrement.down.sql": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa", + "mysql/main/008_job_id_autoincrement.up.sql": "8b228ba007bc109e511c722133dd06f65528fa9cb8c545dbcda2442dc9187bfa", "postgresql/main/001_create_river_migration.down.sql": "34c87dc594bf7520bc3ae69f6f0da8d2d9a472616ab38b37e63d4e3838da06d2", "postgresql/main/001_create_river_migration.up.sql": "79def9ab1643beee7776c499559ec199a03b5b26036c122dc3ba13ec3d078dc0", "postgresql/main/002_initial_schema.down.sql": "8e7e73755b3e9cd1d46f0dffeadd427b86af13cea2f41f3d30af1624329db9b9", diff --git a/ruby/migration/mysql/main/001_create_river_migration.down.sql b/ruby/migration/mysql/main/001_create_river_migration.down.sql new file mode 100644 index 000000000..d2f6b3fab --- /dev/null +++ b/ruby/migration/mysql/main/001_create_river_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE /* TEMPLATE: schema */river_migration; diff --git a/ruby/migration/mysql/main/001_create_river_migration.up.sql b/ruby/migration/mysql/main/001_create_river_migration.up.sql new file mode 100644 index 000000000..8a76b2634 --- /dev/null +++ b/ruby/migration/mysql/main/001_create_river_migration.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE /* TEMPLATE: schema */river_migration ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/ruby/migration/mysql/main/002_initial_schema.down.sql b/ruby/migration/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/ruby/migration/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/ruby/migration/mysql/main/002_initial_schema.up.sql b/ruby/migration/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..f841bf89a --- /dev/null +++ b/ruby/migration/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL has no LISTEN/NOTIFY transport, so river_job_notify is omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql b/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/ruby/migration/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql b/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/ruby/migration/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/ruby/migration/mysql/main/004_pending_and_more.down.sql b/ruby/migration/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/ruby/migration/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/ruby/migration/mysql/main/004_pending_and_more.up.sql b/ruby/migration/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..b544e5000 --- /dev/null +++ b/ruby/migration/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. States use VARCHAR with an +-- explicit constraint, so changing them does not require a table rebuild. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/ruby/migration/mysql/main/005_migration_unique_client.down.sql b/ruby/migration/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..5906e5567 --- /dev/null +++ b/ruby/migration/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/ruby/migration/mysql/main/005_migration_unique_client.up.sql b/ruby/migration/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..b1e83547d --- /dev/null +++ b/ruby/migration/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/ruby/migration/mysql/main/006_bulk_unique.down.sql b/ruby/migration/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/ruby/migration/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/ruby/migration/mysql/main/006_bulk_unique.up.sql b/ruby/migration/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/ruby/migration/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..1f368209e --- /dev/null +++ b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..82ec71552 --- /dev/null +++ b/ruby/migration/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql b/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/ruby/migration/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql b/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/ruby/migration/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/ruby/scripts/check_coverage.rb b/ruby/scripts/check_coverage.rb index 4d6137758..32fb38345 100644 --- a/ruby/scripts/check_coverage.rb +++ b/ruby/scripts/check_coverage.rb @@ -2,7 +2,7 @@ require "simplecov" -# Both backend jobs must supply every package's report; a missing artifact must +# All backend jobs must supply every package's report; a missing artifact must # not turn into an apparently complete report for only one package or backend. reports = %w[postgres sqlite].product([".", "driver/riverqueue-activerecord", "driver/riverqueue-sequel"]).map do |backend, package| path = "coverage/ci/ruby-coverage-#{backend}/#{package}/coverage/.resultset.json" @@ -10,6 +10,10 @@ path end +mysql_report = "coverage/ci/ruby-coverage-mysql/.resultset.json" +abort "Missing coverage report: #{mysql_report}" unless File.file?(mysql_report) +reports << mysql_report + SimpleCov.collate(reports) do enable_coverage :branch minimum_coverage branch: 100, line: 100 diff --git a/ruby/scripts/sync_migrations.rb b/ruby/scripts/sync_migrations.rb index 9e366eaba..7918557ee 100644 --- a/ruby/scripts/sync_migrations.rb +++ b/ruby/scripts/sync_migrations.rb @@ -11,7 +11,7 @@ root = File.expand_path("..", __dir__) destination = File.join(root, "migration") line = "main" -drivers = {"postgresql" => "riverdriver/riverpgxv5", "sqlite" => "riverdriver/riversqlite"} +drivers = {"mysql" => "riverdriver/rivermysql", "postgresql" => "riverdriver/riverpgxv5", "sqlite" => "riverdriver/riversqlite"} files = {} drivers.each do |backend, directory| diff --git a/ruby/sig/driver.rbs b/ruby/sig/driver.rbs index 624b1a532..956c635db 100644 --- a/ruby/sig/driver.rbs +++ b/ruby/sig/driver.rbs @@ -78,6 +78,15 @@ module River def initialize: (untyped) -> void def close: () -> nil def poll: (Float | Integer) -> Array[untyped] + private + def initial_cursor: () -> Integer + def read_batch: () -> Array[untyped] + end + + class MySQL < SQLite + private + def initial_cursor: () -> Integer + def read_batch: () -> Array[untyped] end end @@ -181,12 +190,15 @@ module River def runtime_job_state_column: () -> String def runtime_json: (untyped) -> untyped def runtime_legacy_subtransactions?: () -> bool + def runtime_locks_rows?: () -> bool + def runtime_unique_violation_class: () -> untyped def runtime_merge_metadata: (untyped) -> untyped def runtime_metadata_equals: (untyped, untyped) -> untyped def runtime_nullable_time: (untyped) -> untyped def runtime_notify: (String, untyped) -> untyped def notification_query: (String) -> untyped def runtime_notification_connection: () -> untyped + def runtime_order_by: (JobListParams, String) -> String def sqlite_insert_nonces: (Array[JobInsertParams]) -> Array[String] def runtime_parse_json: (untyped) -> untyped def runtime_parse_time: (untyped) -> untyped diff --git a/ruby/sig/migrator.rbs b/ruby/sig/migrator.rbs index 165cf7bbd..c5d62ef96 100644 --- a/ruby/sig/migrator.rbs +++ b/ruby/sig/migrator.rbs @@ -40,7 +40,9 @@ module River def execute: (String) -> untyped def existing_versions: () -> Array[Integer] def line_column?: () -> bool + def mysql_statements: (String) -> Array[String] def query: (String) -> untyped + def schema_prefix: () -> String def session: [A] (?lock: bool) { () -> A } -> A def table: () -> String end diff --git a/ruby/sig/mysql_runtime.rbs b/ruby/sig/mysql_runtime.rbs new file mode 100644 index 000000000..8f52e954b --- /dev/null +++ b/ruby/sig/mysql_runtime.rbs @@ -0,0 +1,17 @@ +module River::Driver + module MySQLRuntime + include Runtime + + def job_get_by_id: (untyped) -> untyped + def job_insert_many: (Array[JobInsertParams]) -> Array[[River::JobRow, bool]] + + private + + def mysql_columns: (String) -> String + def mysql_delete: (String, String, String, ?limit: untyped) -> untyped + def mysql_locked_job: (untyped) -> untyped + def mysql_queue_paused: (untyped, bool, untyped) -> untyped + def mysql_runtime_job: (untyped) -> River::JobRow + def mysql_update: (String, String, String, Array[String]) -> untyped + end +end diff --git a/ruby/spec/conformance_spec.rb b/ruby/spec/conformance_spec.rb index d08025afc..7259b07c5 100644 --- a/ruby/spec/conformance_spec.rb +++ b/ruby/spec/conformance_spec.rb @@ -2,19 +2,14 @@ require "spec_helper" -# The fixture-backed checks run once, alongside SQLite, without a Go setup in -# every Postgres matrix entry. -return unless RiverTestDatabase.enabled?(:sqlite) +# Keep the serverless SQLite checks and opt into MySQL with TEST_DATABASE. +return unless RiverTestDatabase.enabled?(:sqlite) || RiverTestDatabase.enabled?(:mysql) +require_relative "support/client_test_database" require_relative "support/conformance_fixtures" -require_relative "support/river_test_schema" require "riverqueue-activerecord" require "riverqueue-sequel" require "riverqueue/testing" -class ConformanceRecord < ActiveRecord::Base - self.abstract_class = true -end - RSpec.describe "Go-generated conformance fixtures" do fixture_source = RiverConformanceFixtures.read("unique_keys") # Preserve raw argument tokens, including duplicate keys and negative zero. @@ -131,24 +126,16 @@ class ConformanceRecord < ActiveRecord::Base expect(River::RESUMABLE_STEP_METADATA_KEY).to eq(keys.fetch("resumable_step")) end - # These exercise actual insertion/cancellation and the SQL outbox in both - # drivers without requiring a server. Postgres delivery/commit ordering is - # separately exercised by the shared driver specs. - %w[activerecord sequel].each do |adapter| - context "#{adapter} persisted protocol" do + # Exercise the same persisted protocol on both outbox backends. Postgres + # delivery/commit ordering is separately exercised by the shared driver specs. + %w[active_record sequel].product(%i[mysql sqlite]).each do |adapter, backend| + context "#{adapter} #{backend} persisted protocol", database: backend do around do |example| - if adapter == "activerecord" - ConformanceRecord.establish_connection(adapter: "sqlite3", database: ":memory:") - @driver = River::Driver::ActiveRecord.new(connection_class: ConformanceRecord) - else - database = Sequel.sqlite - @driver = River::Driver::Sequel.new(database) + ClientTestDatabase.public_send("with_#{adapter}", backend) do |driver| + @driver = driver + @client = River::Client.new(@driver) + example.run end - RiverTestSchema.load(@driver) - @client = River::Client.new(@driver) - example.run - ensure - (adapter == "activerecord") ? ConformanceRecord.remove_connection : database&.disconnect end %w[cancel insert metadata_changed pause request_resign resigned resume].each do |name| diff --git a/ruby/spec/driver_runtime_shared_examples.rb b/ruby/spec/driver_runtime_shared_examples.rb index cfd74646a..acb1e3e60 100644 --- a/ruby/spec/driver_runtime_shared_examples.rb +++ b/ruby/spec/driver_runtime_shared_examples.rb @@ -67,6 +67,8 @@ def to_json = JSON.dump(value: @value) let(:now) { Time.utc(2026, 1, 2, 3, 4, 5) } it "rescues a job whose JSON rescue counter overflows a Ruby float" do + skip "MySQL rejects numbers outside the JSON numeric range" if driver.migration_backend == :mysql + inserted = client.insert(DriverRuntimeArgs.new(1), scheduled_at: now - 120).job encoded = driver.send(:runtime_json, '{"river:rescue_count":1e999}') driver.send(:runtime_execute, "UPDATE river_job SET metadata = #{encoded} WHERE id = #{inserted.id}") @@ -80,6 +82,8 @@ def to_json = JSON.dump(value: @value) end it "snoozes a job whose JSON snooze counter overflows a Ruby float" do + skip "MySQL rejects numbers outside the JSON numeric range" if driver.migration_backend == :mysql + worker = Object.new worker.define_singleton_method(:work) { |_job| raise River.job_snooze(60) } client = River::Client.new(driver, config: River::Config.new(workers: River::Workers.new.add("driver_runtime", worker))) @@ -383,7 +387,7 @@ def work(_job) row = insert_job(1, scheduled_at: now - 1) driver.job_claim(id: row.id, attempted_by: "test", now: now) cancel_before_write = true - driver.define_singleton_method(:runtime_returning_ids) do |sql| + driver.define_singleton_method(:runtime_query_rows) do |sql| if cancel_before_write cancel_before_write = false job_cancel(row.id) diff --git a/ruby/spec/driver_shared_examples.rb b/ruby/spec/driver_shared_examples.rb index d80b9e283..fdd8ec27b 100644 --- a/ruby/spec/driver_shared_examples.rb +++ b/ruby/spec/driver_shared_examples.rb @@ -158,8 +158,13 @@ class SimpleArgsWithInsertOpts < SimpleArgs it "does not clean up a finalized job retried before the cleanup write" do job = client.insert(SimpleArgs.new(job_num: 1)).job client.job_update(job.id, River::JobUpdateParams.new(finalized_at: Time.now.utc - 120, state: River::JOB_STATE_COMPLETED)) + retried = false driver.define_singleton_method(:runtime_query_rows) do |sql| - job_retry(job.id) if sql.start_with?("DELETE FROM river_job") + if !retried && (sql.start_with?("DELETE FROM river_job") || + (migration_backend == :mysql && sql.start_with?("SELECT id FROM river_job WHERE"))) + retried = true + job_retry(job.id) + end super(sql) end @@ -167,6 +172,12 @@ class SimpleArgsWithInsertOpts < SimpleArgs expect(client.job_get(job.id).state).to eq(River::JOB_STATE_AVAILABLE) end + it "preserves exact queue names" do + names = ["queue", "queue ", "QUEUE", "cafĂ©", "cafe"] + names.each { |name| expect(driver.queue_upsert(name).name).to eq(name) } + names.each { |name| expect(driver.queue_get(name).name).to eq(name) } + end + it "keeps static driver constants shareable across Ractor boundaries" do values = %i[SQLITE_CONFLICT_WHERE SQLITE_JOB_COLUMNS UNIQUE_INSERT_METADATA_KEY] .map { |name| driver.class.const_get(name, false) } @@ -568,13 +579,23 @@ class SimpleArgsWithInsertOpts < SimpleArgs stored = inserted.map do |result| driver.send(:runtime_query_rows, <<~SQL).first.values.first - SELECT #{postgres ? "args::text" : "json(args)"} AS args FROM river_job WHERE id = #{result.job.id} + SELECT #{if postgres + "args::text" + else + ((driver.migration_backend == :mysql) ? "CAST(args AS CHAR)" : "json(args)") + end} AS args FROM river_job WHERE id = #{result.job.id} SQL end - expect(stored).to all(eq(postgres ? + expected = postgres ? '{"decimal": 1.10, "unicode": "cafĂ©", "exponent": 1000, "high_precision": 1234567890.1234567890}' : - encoded)) - expect(inserted.map { |result| result.job.args }).to all(eq(JSON.parse(encoded))) + encoded + if driver.migration_backend == :mysql + # MySQL normalizes its native JSON representation, including numbers. + literal = driver.send(:runtime_quote, encoded) + expected = driver.send(:runtime_query_rows, "SELECT CAST(CAST(#{literal} AS JSON) AS CHAR) AS args").first.values.first + end + expect(stored).to all(eq(expected)) + expect(inserted.map { |result| result.job.args }).to all(eq(JSON.parse(expected))) end end diff --git a/ruby/spec/migration_export_spec.rb b/ruby/spec/migration_export_spec.rb index a8fbbdfd5..46cb4866c 100644 --- a/ruby/spec/migration_export_spec.rb +++ b/ruby/spec/migration_export_spec.rb @@ -34,6 +34,12 @@ def versions expect(stdout).not_to include("TEMPLATE") end + it "exports MySQL SQL with schema substitution without connecting" do + expect(run("--version", "2", "--up", "--database-url", "mysql://invalid.invalid", "--schema", "jobs")).to eq(0), err.string + expect(out.string).to include("CREATE TABLE `jobs`.river_job", "utf8mb4_0900_bin") + expect(out.string).not_to include("TEMPLATE") + end + it "exports no SQL when every selected version is excluded" do expect(run("--version", "1", "--exclude-version", "1", "--up")).to eq(0) expect(out.string).to eq("") @@ -118,7 +124,7 @@ class Migrator < River::Migrator [["--all", "--up", "--down"], "pass exactly one of --up or --down"], [["--version", "1,999", "--up"], "migration 999 does not exist"], [["--version", "1,999", "--exclude-version", "999", "--up"], "migration 999 does not exist"], - [["--all", "--up", "--database-url", "mysql://user:secret@localhost"], "--database-url must use"], + [["--all", "--up", "--database-url", "unknown://user:secret@localhost"], "--database-url must use"], [["--all", "--up", "--database-url", "sqlite://", "--schema", "jobs"], "SQLite does not support"], [["--all", "--up", "--schema", 'bad"schema'], "schema must be a simple SQL identifier"], [["--all", "--up", "--dry-run"], "--dry-run cannot be used with migrate-get"], diff --git a/ruby/spec/migrator_shared_examples.rb b/ruby/spec/migrator_shared_examples.rb index 5e2e1c018..debb2864d 100644 --- a/ruby/spec/migrator_shared_examples.rb +++ b/ruby/spec/migrator_shared_examples.rb @@ -68,7 +68,11 @@ migrator.migrate(target: 6) @driver.migration_connection do |connection| sql = "INSERT INTO river_job (args, kind, max_attempts, metadata) VALUES ('{\"value\":42}', 'migration_test', 25, '{\"keep\":true}')" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end migrator.migrate @@ -105,13 +109,21 @@ migrator.migrate @driver.migration_connection do |connection| sql = "INSERT INTO river_migration (line, version) VALUES ('main', 9)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.migrate }.to raise_error(River::Error, /history/) @driver.migration_connection do |connection| sql = "DELETE FROM river_migration WHERE version IN (3, 9)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.status }.to raise_error(River::Error, /history/) @@ -121,7 +133,11 @@ migrator.migrate @driver.migration_connection do |connection| sql = "INSERT INTO river_migration (line, version) VALUES ('pro', 1)" - (@driver.migration_backend == :postgresql) ? connection.exec(sql) : connection.execute_batch(sql) + case @driver.migration_backend + when :mysql then connection.query(sql) + when :postgresql then connection.exec(sql) + else connection.execute_batch(sql) + end end expect { migrator.migrate(direction: :down, target: 4) }.to raise_error(River::Error, /non-main/) diff --git a/ruby/spec/migrator_spec.rb b/ruby/spec/migrator_spec.rb index 489a2562c..2250ee9fd 100644 --- a/ruby/spec/migrator_spec.rb +++ b/ruby/spec/migrator_spec.rb @@ -29,7 +29,84 @@ expect { described_class.new(driver, schema: "other") }.to raise_error(ArgumentError, /SQLite/) end - [:postgres, :sqlite].each do |adapter| + context "MySQL migration session without a server" do + let(:connection) do + Class.new do + attr_accessor :fail_ddl, :locked, :schema + attr_reader :statements, :versions + + def initialize + @locked = 1 + @schema = "test_db" + @statements = [] + @versions = [] + end + + def query(sql, **) + @statements << sql + case sql + when /SELECT DATABASE/ then [{"name" => schema}] + when /GET_LOCK/ then [{"locked" => locked}] + when /information_schema.tables/ then versions.empty? ? [] : [{"table_name" => "river_migration"}] + when /information_schema.columns/ then [] + when /SELECT version/ then versions.map { |version| {"version" => version} } + when /CREATE TABLE/ + raise "DDL failed" if fail_ddl + [] + when /INSERT INTO .*river_migration.*VALUES \((\d+)\)/ + versions << Regexp.last_match(1).to_i + [] + else [] + end + end + end.new + end + let(:driver) do + value = connection + Object.new.tap do |driver| + driver.define_singleton_method(:migration_backend) { :mysql } + driver.define_singleton_method(:migration_connection) { |&block| block.call(value) } + end + end + + it "records history after DDL and releases the lock without transaction commands" do + migrator = described_class.new(driver) + expect(migrator.migrate(steps: 1).map(&:version)).to eq([1]) + expect(connection.versions).to eq([1]) + expect(connection.statements.last).to include("RELEASE_LOCK") + expect(connection.statements).not_to include("BEGIN", "COMMIT", "ROLLBACK") + expect(migrator.status.first.applied).to be true + end + + it "releases its lock after a DDL failure without recording the failed version" do + connection.fail_ddl = true + expect { described_class.new(driver).migrate(steps: 1) }.to raise_error("DDL failed") + expect(connection.versions).to be_empty + expect(connection.statements.last).to include("RELEASE_LOCK") + end + + it "leaves another migrator's lock alone" do + connection.locked = 0 + expect { described_class.new(driver).migrate }.to raise_error(River::Error, /database lock/) + expect(connection.statements.grep(/RELEASE_LOCK|CREATE TABLE/)).to be_empty + end + + it "requires a selected safe database and supports an explicit schema" do + [nil, "bad'name", "a" * 65].each do |schema| + connection.schema = schema + expect { described_class.new(driver).status }.to raise_error(ArgumentError, /identifier/) + end + described_class.new(driver, schema: "other").migrate(steps: 1) + expect(connection.statements.grep(/CREATE TABLE/).first).to include("`other`.river_migration") + end + + it "splits migration scripts while preserving quoted semicolons" do + migrator = described_class.new(driver) + expect(migrator.send(:mysql_statements, "; SELECT 'a;b'; -- comment;\n SELECT \"x;y\"; /* ; */ SELECT `a;b` # comment;\n")).to eq(["SELECT 'a;b'", "SELECT \"x;y\"", "SELECT `a;b`"]) + end + end + + [:mysql, :postgres, :sqlite].each do |adapter| context "with #{adapter}", database: adapter do around do |example| skip "Postgres unavailable" if adapter == :postgres && !DB @@ -57,6 +134,43 @@ expect { migrator.migrate }.to raise_error(River::Error, /changed concurrently/) end + if adapter == :mysql + it "supports explicit schemas and rejects unsafe identifiers" do + schema = @driver.send(:runtime_query_rows, "SELECT DATABASE() AS name").first.fetch(:name) + expect(described_class.new(@driver, schema: schema).migrate.length).to eq(8) + expect { described_class.new(@driver, schema: "bad'name").status }.to raise_error(ArgumentError, /identifier/) + end + + it "migrates databases with the maximum 64-character name" do + schema = "river_migrate_#{SecureRandom.hex(16)}".ljust(64, "x") + @driver.send(:runtime_execute, "CREATE DATABASE `#{schema}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin") + migrator = described_class.new(@driver, schema: schema) + expect(migrator.migrate.length).to eq(8) + expect(migrator.migrate(direction: :down, steps: 8).length).to eq(8) + ensure + @driver.send(:runtime_execute, "DROP DATABASE IF EXISTS `#{schema}`") if schema + end + + it "refuses concurrent migrators and releases its named lock" do + schema = @driver.send(:runtime_query_rows, "SELECT DATABASE() AS name").first.fetch(:name) + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL")) + admin.fetch("SELECT GET_LOCK(CONCAT('river_migrate:',LEFT(SHA2('#{schema}',256),48)), 0)").all + expect { described_class.new(@driver).migrate }.to raise_error(River::Error, /database lock/) + admin.fetch("SELECT RELEASE_LOCK(CONCAT('river_migrate:',LEFT(SHA2('#{schema}',256),48)))").all + expect(described_class.new(@driver).migrate.length).to eq(8) + ensure + admin&.disconnect + end + + it "requires a selected database" do + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL").sub(%r{/[^/]*$}, "/")) + expect { described_class.new(River::Driver::Sequel.new(admin)).status }.to raise_error(ArgumentError, /identifier/) + ensure + admin&.disconnect + end + + end + if adapter == :postgres it "rejects an empty search path" do @driver.migration_connection do |connection| diff --git a/ruby/spec/mysql_spec.rb b/ruby/spec/mysql_spec.rb new file mode 100644 index 000000000..fe387a6d7 --- /dev/null +++ b/ruby/spec/mysql_spec.rb @@ -0,0 +1,177 @@ +# frozen_string_literal: true + +require "spec_helper" +require "active_record" +require "sequel" +require "riverqueue-activerecord" +require "riverqueue-sequel" +require_relative "driver_shared_examples" +require_relative "row_decoding_shared_examples" +require_relative "support/client_test_database" + +RSpec.describe "MySQL drivers", database: :mysql do + [:active_record, :sequel].each do |adapter| + context adapter.to_s do + before(:all) do + @admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL", "mysql2://root@127.0.0.1/")) + @schema = "river_ruby_mysql_#{SecureRandom.hex(8)}" + @admin.run("CREATE DATABASE #{@schema}") + config = @admin.opts.merge(database: @schema, max_connections: 10) + if adapter == :sequel + @database = Sequel.connect(config) + @driver = River::Driver::Sequel.new(@database) + else + @original_connection = ClientTestDatabase.active_record_config + ActiveRecord::Base.establish_connection(adapter: "mysql2", host: config[:host], port: config[:port], + username: config[:user], password: config[:password], database: @schema, pool: 10) + @driver = River::Driver::ActiveRecord.new + end + River::Migrator.new(@driver).migrate + end + + after(:all) do + @database&.disconnect + if adapter == :active_record + if @original_connection + ActiveRecord::Base.establish_connection(@original_connection) + else + ActiveRecord::Base.remove_connection + end + end + @admin&.run("DROP DATABASE #{@schema}") if @schema + @admin&.disconnect + end + + around(:each) do |example| + driver.transaction do + example.run + raise driver.rollback_exception + end + end + + let(:driver) { (adapter == :sequel) ? River::Driver::Sequel.new(@database) : River::Driver::ActiveRecord.new } + let(:client) { River::Client.new(driver) } + + it_behaves_like "driver shared examples" + it_behaves_like "historical attempt error decoding" + + it "handles empty batches and uses the database clock for an omitted schedule" do + expect(driver.job_insert_many([])).to eq([]) + params = River::Driver::JobInsertParams.new(encoded_args: "{}", kind: "simple", max_attempts: 25, + priority: 1, queue: "default", scheduled_at: nil, state: "available", tags: nil) + row, duplicate = driver.job_insert(params) + expect(duplicate).to be(false) + expect(row.scheduled_at).to be_within(2).of(Time.now.getutc) + expect { driver.send(:runtime_time, nil) }.to raise_error(ArgumentError, /nil/) + end + + it "appends scalar attempt errors without changing their JSON type" do + row = client.insert(SimpleArgs.new(job_num: 1)).job + sql = driver.send(:runtime_append_error, JSON.generate("a scalar error")) + driver.send(:runtime_execute, "UPDATE river_job SET errors = #{sql} WHERE id = #{row.id}") + expect(driver.job_get_by_id(row.id).errors).to contain_exactly(have_attributes(error: "a scalar error")) + end + + it "matches kinds and queues exactly, including trailing spaces" do + wanted = client.insert(SimpleArgs.new(job_num: 1)).job + padded = client.insert(SimpleArgs.new(job_num: 2)).job + driver.send(:runtime_execute, "UPDATE river_job SET kind = 'simple ', queue = 'default ' WHERE id = #{padded.id}") + expect(client.job_list(River::JobListParams.new(kinds: ["simple"])).jobs.map(&:id)).to eq([wanted.id]) + expect(client.job_list(River::JobListParams.new(queues: ["default"])).jobs.map(&:id)).to eq([wanted.id]) + expect(driver.job_get_available(queue: "default", max: 10, attempted_by: "test").map(&:id)).to eq([wanted.id]) + end + + it "keeps adjacent large IDs distinct" do + first = client.insert(SimpleArgs.new(job_num: 1)).job + second = client.insert(SimpleArgs.new(job_num: 2)).job + driver.send(:runtime_execute, "UPDATE river_job SET id = #{2**53} WHERE id = #{first.id}") + driver.send(:runtime_execute, "UPDATE river_job SET id = #{2**53 + 1} WHERE id = #{second.id}") + expect(client.job_list(River::JobListParams.new(ids: [2**53 + 1])).jobs.map(&:id)).to eq([2**53 + 1]) + end + end + end +end + +require_relative "client_driver_shared_examples" + +RSpec.describe "MySQL worker integration", database: :mysql do + [:active_record, :sequel].each do |adapter| + context adapter.to_s do + around do |example| + ClientTestDatabase.public_send("with_#{adapter}", :mysql) do |driver| + @driver = driver + example.run + end + end + + it_behaves_like "client driver end to end" + + it "rejects conflicts on an unrelated unique index" do + @driver.send(:runtime_execute, "CREATE UNIQUE INDEX mysql_test_kind ON river_job(kind)") + client = River::Client.new(@driver) + client.insert(SimpleArgs.new(job_num: 1)) + expect { client.insert(SimpleArgs.new(job_num: 2)) }.to raise_error(River::Error, /other than River/) + expect { client.insert(SimpleArgs.new(job_num: 3), unique_opts: River::UniqueOpts.new(by_args: true)) }.to raise_error(River::Error, /other than River/) + end + + it "rejects unrelated conflicts with inactive unique keys" do + @driver.send(:runtime_execute, "CREATE UNIQUE INDEX mysql_test_kind ON river_job(kind)") + client = River::Client.new(@driver) + unique = River::UniqueOpts.new(by_args: true, by_state: [:available, :pending, :running, :scheduled]) + params = client.send(:make_insert_params, SimpleArgs.new(job_num: 1), River::InsertOpts.new(unique_opts: unique)) + job, = @driver.job_insert(params) + params.state = "retryable" + expect { @driver.job_insert(params) }.to raise_error(River::Error, /other than River/) + + @driver.send(:runtime_execute, "UPDATE river_job SET unique_states = NULL WHERE id = #{job.id}") + params.state = "available" + expect { @driver.job_insert(params) }.to raise_error(River::Error, /other than River/) + end + + it "migrates down and back up using the Go migration history" do + migrator = River::Migrator.new(@driver) + expect(migrator.status).to all(have_attributes(applied: true)) + migrator.migrate(direction: :down, target: 0) + expect(migrator.status).to all(have_attributes(applied: false)) + migrator.migrate + expect(migrator.status).to all(have_attributes(applied: true)) + end + end + end +end + +RSpec.describe "MySQL cleaner", database: :mysql do + [:active_record, :sequel].each do |adapter| + it "skips locked jobs with #{adapter}" do + ClientTestDatabase.public_send("with_#{adapter}", :mysql) do |driver| + params = River::Driver::JobInsertParams.new(encoded_args: "{}", kind: "cleanup", max_attempts: 25, + priority: 1, queue: "default", scheduled_at: nil, state: "available", tags: []) + running, locked, *expired = 1002.times.map { driver.job_insert(params).first.id } + driver.send(:runtime_execute, "UPDATE river_job SET state = 'completed', finalized_at = '1970-01-01' WHERE id >= #{locked}") + driver.send(:runtime_execute, "UPDATE river_job SET state = 'running' WHERE id = #{running}") + ready = Queue.new + release = Queue.new + locker = Thread.new do + driver.transaction do + [running, locked].each do |id| + driver.send(:runtime_query_rows, "SELECT id FROM river_job WHERE id = #{id} FOR UPDATE") + end + ready << true + release.pop + end + end + begin + Timeout.timeout(5) { ready.pop } + deleted = Timeout.timeout(2) { driver.job_delete_finalized(retention: {completed: 0}, max: 1000) } + expect(deleted).to eq(1000) + expect(driver.job_get_by_id(expired.first)).to be_nil + expect(driver.job_get_by_id(locked).state).to eq("completed") + expect(driver.job_get_by_id(running).state).to eq("running") + ensure + release << true + locker.join + end + end + end + end +end diff --git a/ruby/spec/notification_receiver_shared_examples.rb b/ruby/spec/notification_receiver_shared_examples.rb index 6be80aed4..01206d3c7 100644 --- a/ruby/spec/notification_receiver_shared_examples.rb +++ b/ruby/spec/notification_receiver_shared_examples.rb @@ -114,7 +114,8 @@ def receiver_client(**options, &work) @driver.transaction do client.request_resign - expect(Thread.new { first.poll(0.01) }.value).to be_empty + # MySQL locking reads wait for the transaction, so read after commit. + expect(Thread.new { first.poll(0.01) }.value).to be_empty unless backend == :mysql end expected = [["river_leadership", {"action" => "request_resign", "leader_id" => ""}]] @@ -130,6 +131,46 @@ def receiver_client(**options, &work) second&.close end + if backend == :mysql + [false, true].each do |during_subscribe| + it "waits for lower uncommitted notification IDs #{during_subscribe ? "when subscribing" : "when polling"}" do + listener = @driver.notification_listener unless during_subscribe + inserted = Queue.new + release = Queue.new + writer = Thread.new do + @driver.transaction do + @driver.send(:runtime_notify, "river_insert", queue: "first") + inserted << true + release.pop + end + end + receive_signal(inserted) + @driver.send(:runtime_notify, "river_insert", queue: "second") + reader = Thread.new do + during_subscribe ? @driver.notification_listener : listener.poll(0.01) + end + expect(reader.join(0.05)).to be_nil + release << true + writer.value + if during_subscribe + listener = reader.value + @driver.send(:runtime_notify, "river_insert", queue: "new") + expect(read_notifications(listener)).to eq([["river_insert", {"queue" => "new"}]]) + else + expect(reader.value.map { |topic, payload| [topic, JSON.parse(payload)] }).to eq([ + ["river_insert", {"queue" => "first"}], + ["river_insert", {"queue" => "second"}] + ]) + end + ensure + release << true if release + writer&.join + reader&.join + listener&.close + end + end + end + it "fails startup if subscribing fails and allows a subsequent start" do original = @driver.method(:notification_listener) attempts = 0 diff --git a/ruby/spec/notification_receiver_spec.rb b/ruby/spec/notification_receiver_spec.rb index c1c4d014d..cffc3e51c 100644 --- a/ruby/spec/notification_receiver_spec.rb +++ b/ruby/spec/notification_receiver_spec.rb @@ -8,7 +8,7 @@ RSpec.describe "River notification receiver" do %i[active_record sequel].each do |adapter| - %i[postgres sqlite].each do |backend| + %i[mysql postgres sqlite].each do |backend| context "#{adapter} with #{backend}", database: backend do around do |example| ClientTestDatabase.public_send(:"with_#{adapter}", backend) do |driver| diff --git a/ruby/spec/notification_runtime_spec.rb b/ruby/spec/notification_runtime_spec.rb index d3e6e6d57..e51135b65 100644 --- a/ruby/spec/notification_runtime_spec.rb +++ b/ruby/spec/notification_runtime_spec.rb @@ -60,6 +60,29 @@ runtime.stop(cancel: true) end + [[], [{id: 4}]].each do |history| + it "preserves a MySQL outbox cursor across failed reads with #{history.length} historical rows" do + queries = [] + responses = [history, RuntimeError.new("connection lost"), [{id: 10, topic: "river_insert", payload: "{}"}], []] + query = lambda do |sql| + queries << sql + response = responses.shift + raise response if response.is_a?(Exception) + + response + end + listener = River::Driver::NotificationListener::MySQL.new(query) + expect { listener.poll(0) }.to raise_error("connection lost") + expect(listener.poll(0)).to eq([["river_insert", "{}"]]) + expect(listener.poll(0)).to be_empty + expect(queries[1]).to eq(queries[2]) + expect(queries[1]).to include("id > #{history.empty? ? 0 : 4}") + expect(queries[3]).to include("id > 10") + ensure + listener&.close + end + end + it "preserves maintenance wakeups received before the elector starts waiting" do condition = runtime.instance_variable_get(:@maintenance_condition) condition.define_singleton_method(:wait) { |*_args| raise "unexpected maintenance wait" } diff --git a/ruby/spec/row_decoding_shared_examples.rb b/ruby/spec/row_decoding_shared_examples.rb index dd29e89d8..b1712eeee 100644 --- a/ruby/spec/row_decoding_shared_examples.rb +++ b/ruby/spec/row_decoding_shared_examples.rb @@ -1,6 +1,18 @@ # frozen_string_literal: true RSpec.shared_examples "historical attempt error decoding" do + it "completes an attempt when stored metadata becomes unreadable" do + row = client.insert(River::JobArgsHash.new(:decode, {})).job + claimed = driver.job_claim(id: row.id, attempted_by: "test") + encoded = driver.send(:runtime_json, "[]") + driver.send(:runtime_execute, "UPDATE river_job SET metadata = #{encoded} WHERE id = #{row.id}") + + completed = driver.job_complete(id: row.id, finalized_at: Time.now.utc, expected_job: claimed) + + expect(completed).to have_attributes(state: "completed", __decode_error: be_a(River::JobRowDecodeError)) + expect(driver.send(:runtime_read_job, row.id).state).to eq("completed") + end + it "isolates non-object metadata across reads, unique inserts, claims, and finalization" do rows = [[], "invalid metadata", 7, true].map.with_index do |metadata, index| args = River::JobArgsHash.new(:decode, {index: index}) diff --git a/ruby/spec/spec_helper.rb b/ruby/spec/spec_helper.rb index 17307e844..b291f8e5c 100644 --- a/ruby/spec/spec_helper.rb +++ b/ruby/spec/spec_helper.rb @@ -16,9 +16,13 @@ merging false if RiverTestDatabase::BACKEND == "all" # Drivers have their own spec suite where they're covered 100.0%, but - # they're not fully covered from this top level test suite. - skip("driver/riverqueue-sequel/") - skip("driver/riverqueue-activerecord/") + # they're not fully covered from this top level test suite. MySQL's + # adapter coverage is collected here alongside its shared driver tests. + unless RiverTestDatabase::BACKEND == "mysql" + skip("driver/riverqueue-sequel/") + skip("driver/riverqueue-activerecord/") + end + skip("/lib/driver/mysql_runtime.rb") unless RiverTestDatabase.enabled?(:mysql) skip("/spec/") end end diff --git a/ruby/spec/support/client_test_database.rb b/ruby/spec/support/client_test_database.rb index 7efd99b85..6e3028cf8 100644 --- a/ruby/spec/support/client_test_database.rb +++ b/ruby/spec/support/client_test_database.rb @@ -8,9 +8,26 @@ # rollback-wrapped driver contracts. Never run their workers against public # tables: migrate an empty disposable schema with the bundled canonical SQL. module ClientTestDatabase + def self.active_record_config + ActiveRecord::Base.connection_handler.retrieve_connection_pool(ActiveRecord::Base.connection_specification_name)&.db_config&.configuration_hash + end + def self.with_active_record(adapter, migrate: true, pg_catalog_last: false) # The core suite may not have configured a connection yet. original = ActiveRecord::Base.remove_connection + if adapter == :mysql + return with_mysql do |config| + ActiveRecord::Base.establish_connection(adapter: "mysql2", host: config[:host], port: config[:port], + username: config[:user], password: config[:password], database: config[:database], pool: 20) + begin + driver = River::Driver::ActiveRecord.new + RiverTestSchema.load(driver) if migrate + yield driver + ensure + ActiveRecord::Base.connection_pool.disconnect! + end + end + end if adapter == :postgres ActiveRecord::Base.establish_connection(ENV["TEST_DATABASE_URL"] || "postgres://localhost/river_test") schema = "river_client_test_#{SecureRandom.hex(8)}" @@ -49,6 +66,18 @@ def self.with_active_record(adapter, migrate: true, pg_catalog_last: false) end def self.with_sequel(adapter, migrate: true, pg_catalog_last: false) + if adapter == :mysql + return with_mysql do |config| + database = Sequel.connect(config.merge(max_connections: 20)) + begin + driver = River::Driver::Sequel.new(database) + RiverTestSchema.load(driver) if migrate + yield driver + ensure + database.disconnect + end + end + end if adapter == :postgres schema = "river_client_test_#{SecureRandom.hex(8)}" search_path = pg_catalog_last ? "#{schema},pg_catalog" : "#{schema},public" @@ -79,4 +108,14 @@ def self.with_sequel(adapter, migrate: true, pg_catalog_last: false) database&.disconnect end end + + def self.with_mysql + admin = Sequel.connect(ENV.fetch("TEST_MYSQL_URL", "mysql2://root@127.0.0.1/")) + schema = "river_ruby_mysql_#{SecureRandom.hex(8)}" + admin.run("CREATE DATABASE #{schema}") + yield admin.opts.merge(database: schema) + ensure + admin&.run("DROP DATABASE #{schema}") if schema + admin&.disconnect + end end diff --git a/ruby/spec/support/migrations/mysql/test/001_test.down.sql b/ruby/spec/support/migrations/mysql/test/001_test.down.sql new file mode 100644 index 000000000..bfc26b8ca --- /dev/null +++ b/ruby/spec/support/migrations/mysql/test/001_test.down.sql @@ -0,0 +1 @@ +DROP TABLE river_migration_test; diff --git a/ruby/spec/support/migrations/mysql/test/001_test.up.sql b/ruby/spec/support/migrations/mysql/test/001_test.up.sql new file mode 100644 index 000000000..d985fb052 --- /dev/null +++ b/ruby/spec/support/migrations/mysql/test/001_test.up.sql @@ -0,0 +1 @@ +CREATE TABLE river_migration_test (id integer PRIMARY KEY); diff --git a/ruby/spec/support/river_test_schema.rb b/ruby/spec/support/river_test_schema.rb index 8a3388125..048f18163 100644 --- a/ruby/spec/support/river_test_schema.rb +++ b/ruby/spec/support/river_test_schema.rb @@ -5,6 +5,8 @@ # and advisory locks. Replay the same SQL, without maintaining a schema copy. module RiverTestSchema def self.load(driver) + return River::Migrator.new(driver).migrate if driver.migration_backend == :mysql + migrations = River::Migrator.new(driver).migrations sql = migrations.map do |migration| # Postgres must commit new enum values before later migrations use them. diff --git a/ruby/spec/support/test_database.rb b/ruby/spec/support/test_database.rb index b873d7417..207b4933e 100644 --- a/ruby/spec/support/test_database.rb +++ b/ruby/spec/support/test_database.rb @@ -3,14 +3,15 @@ # Like the Go suite, default to both backends and allow CI to test them separately. module RiverTestDatabase BACKEND = ENV.fetch("TEST_DATABASE", "all") - raise ArgumentError, "TEST_DATABASE must be all, postgres, or sqlite" unless %w[all postgres sqlite].include?(BACKEND) + raise ArgumentError, "TEST_DATABASE must be all, mysql, postgres, or sqlite" unless %w[all mysql postgres sqlite].include?(BACKEND) def self.enabled?(backend) + return BACKEND == "mysql" || (BACKEND == "all" && ENV.key?("TEST_MYSQL_URL")) if backend == :mysql + BACKEND == "all" || backend.to_s == BACKEND end end RSpec.configure do |config| - config.filter_run_excluding database: :postgres unless RiverTestDatabase.enabled?(:postgres) - config.filter_run_excluding database: :sqlite unless RiverTestDatabase.enabled?(:sqlite) + config.filter_run_excluding database: ->(backend) { backend && !RiverTestDatabase.enabled?(backend) } end diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index fa43bf6a7..2167d0e9a 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -10,6 +10,10 @@ Changes to River for Go are recorded in the [repository changelog](../CHANGELOG. ## [Unreleased] +### Added + +- Add MySQL support through the `mysql` feature, including transactional job operations, workers, notification delivery, maintenance, test helpers, and CLI migrations. + ## [0.6.0] - 2026-11-05 ### Added diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 58a12703f..639db4c01 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -968,6 +968,7 @@ dependencies = [ "serde_json", "sha2 0.11.1", "sqlx", + "sqlx-core", "thiserror", "tokio", "tokio-util", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 233b25992..dd575cf14 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,6 +29,7 @@ serde = { version = "1.0.229", features = ["derive"] } serde_json = { version = "1.0.151", features = ["raw_value"] } sha2 = "0.11.0" sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio"] } +sqlx-core = { version = "0.9.0", default-features = false } syn = { version = "3.0", features = ["full"] } thiserror = "2.0.20" tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] } @@ -40,7 +41,7 @@ missing_debug_implementations = "warn" unsafe_code = "forbid" # `--cfg river_postgres_tests` builds the Postgres integration tests, which # need `RIVER_RUST_DATABASE_URL`; `make test/rust` sets it when the URL is set. -unexpected_cfgs = { level = "warn", check-cfg = ["cfg(river_postgres_tests)"] } +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(river_postgres_tests)", "cfg(river_mysql_tests)"] } [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/rust/README.md b/rust/README.md index 9663ac6cb..4670d6a8a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,6 +1,6 @@ # River for Rust -River is a fast, reliable background job system backed by Postgres or SQLite. Its Rust implementation uses Tokio and shares River's database schema and job protocol with the other languages, so services can insert and work jobs in the same database. +River is a fast, reliable background job system backed by Postgres, MySQL, or SQLite. Its Rust implementation uses Tokio and shares River's database schema and job protocol with the other languages, so services can insert and work jobs in the same database. ## Installation @@ -18,6 +18,7 @@ Rust version is 1.95. | Feature | Default | Enables | |---|---|---| +| `mysql` | no | MySQL 8.0 or newer through SQLx | | `postgres` | yes | Postgres through SQLx | | `sqlite` | no | SQLite 3.45 or newer through SQLx | | `tzdb` | no | IANA zone names such as `America/New_York` in cron `CRON_TZ=` and `TZ=` prefixes | @@ -35,6 +36,41 @@ compatible with River's (SQLx 0.9, Jiff 0.2, `serde_json` 1, and implementation for SQLx; enable one of SQLx's TLS features in your own SQLx dependency if your database connections use TLS. +## MySQL + +Enable the `mysql` feature, and pass a SQLx `MySqlPool` to the client: + +```toml +riverqueue = { version = "0.6.0", default-features = false, features = ["mysql"] } +``` + +```rust,no_run +# #[cfg(feature = "mysql")] +# async fn example() -> Result<(), Box> { +use riverqueue::{Client, migrate::MySqlMigrator, sqlx::MySqlPool}; + +let pool = MySqlPool::connect("mysql://localhost/app").await?; +MySqlMigrator::new(pool.clone()).migrate_up().await?; +let client = Client::builder(pool).build()?; +# Ok(()) +# } +``` + +Create the database before migrating. Jobs, workers, queues, and `.tx(...)` +requests use the same APIs as the other backends. MySQL clients poll River's +transactional notification outbox for insertions, queue changes, cancellation, +and leadership requests. River's own transactions use READ COMMITTED isolation; +caller-owned transactions retain their configured isolation. Use +`riverqueue::database::begin_mysql_write` for a cancellation-safe begin at +READ COMMITTED. + +MySQL implicitly commits DDL, so a failed migration may leave successful +statements applied. Migration history advances only after all statements in +that migration succeed. + +See the shared [MySQL design notes](../docs/development.md#mysql-design-and-operation) for JSON numeric limits, +exact string comparisons, and server packet-size requirements. + ## Quick start Set `DATABASE_URL` to your application's Postgres database. Define serializable job arguments, register a worker, apply migrations, and start a client: @@ -105,7 +141,7 @@ Insert jobs in the same transaction as application data by chaining `.tx(&mut tr - [Unique, periodic, and resumable jobs](riverqueue/README.md#reliability-features). - [Event subscriptions](riverqueue/README.md#events) for logging and metrics. - [Leader election and maintenance](riverqueue/README.md#leadership-and-maintenance) for scheduling, rescue, and cleanup. -- [Postgres and SQLite support](riverqueue/README.md#database-support) with caller-owned SQLx pools. +- [MySQL, Postgres, and SQLite support](riverqueue/README.md#database-support) with caller-owned SQLx pools. ## Crates diff --git a/rust/docs/development.md b/rust/docs/development.md index 7e1e971ae..fbf19cf78 100644 --- a/rust/docs/development.md +++ b/rust/docs/development.md @@ -32,6 +32,15 @@ committed. The `make test/rust` targets generate them first, so Go is needed to run the tests; when running `cargo test` directly, run `make generate/fixtures` beforehand. A missing fixture fails its test. +`RIVER_RUST_MYSQL_URL=mysql://root@localhost/ make test/rust/conformance/mysql` +runs the Go fixture checks against MySQL storage: states and unique bits, +attempt errors, rescue/output metadata, snooze counters through real workers, +and notification emission and reception, including rollback. Tests create and +drop their own databases using the canonical migrations. The shared Conformance +workflow runs these checks when Go changes, and `make test/rust/mysql` includes +them in the full MySQL suite. Like Go's MySQL driver, Rust does not emit a +`resigned` notification; the fixture suite checks reception of that message. + `make check/rust/package` builds the five publishable crate archives and verifies that each one builds from its packaged sources, resolving the exact-version workspace dependencies from the other archives. It does not @@ -102,3 +111,16 @@ Run these commands from the repository root. All five Rust crates are versioned ``` 7. Create a [GitHub release](https://github.com/riverqueue/river/releases/new), select `rust/v$VERSION`, and copy the version's `rust/CHANGELOG.md` content into the release body. Mark alpha, beta, and RC versions as prereleases. + +### MySQL + +Run MySQL integration tests with a URL whose user can create temporary databases: + +```sh +RIVER_RUST_MYSQL_URL=mysql://root@localhost/ make test/rust/mysql +``` + +The test runner enables `river_mysql_tests` explicitly, so a missing database +URL fails the target. CI exercises MySQL 8.0 and 8.4. MySQL migration mirrors +come from `riverdriver/rivermysql/migration/main` and are regenerated by +`make generate/rust-migrations`. diff --git a/rust/riverqueue-cli/Cargo.toml b/rust/riverqueue-cli/Cargo.toml index 9efa30bf9..cf1d8eccb 100644 --- a/rust/riverqueue-cli/Cargo.toml +++ b/rust/riverqueue-cli/Cargo.toml @@ -15,8 +15,9 @@ homepage.workspace = true include = ["src/**", "LICENSE", "README.md"] [features] -default = ["postgres", "sqlite"] +default = ["mysql", "postgres", "sqlite"] postgres = ["riverqueue/postgres", "riverqueue-migrate/postgres", "sqlx/postgres"] +mysql = ["riverqueue/mysql", "riverqueue-migrate/mysql", "sqlx/mysql"] sqlite = ["riverqueue/sqlite", "riverqueue-migrate/sqlite", "sqlx/sqlite"] [[bin]] diff --git a/rust/riverqueue-cli/README.md b/rust/riverqueue-cli/README.md index 80b0e3dea..1348eb546 100644 --- a/rust/riverqueue-cli/README.md +++ b/rust/riverqueue-cli/README.md @@ -16,8 +16,12 @@ implementation. Apply them before starting clients: riverqueue migrate-up --database-url postgres://localhost/app riverqueue migrate-up --database-url postgres://localhost/app --schema river riverqueue migrate-up --database-url sqlite://app.sqlite3 +riverqueue migrate-up --database-url mysql://localhost/app ``` +The CLI includes Postgres, MySQL, and SQLite support by default. For MySQL, +select an existing database in the URL; `--schema` is a Postgres option. + `migrate-down`, `migrate-list`, and `validate` take the same connection options. `--target-version N`, `--max-steps N`, and `--dry-run` limit or preview a migration run. Applications can instead migrate from Rust with the @@ -37,7 +41,7 @@ with exit status 2; connection and usage errors exit with status 1. `migrate-get` exports SQL for another migration framework without connecting to a database. It defaults to Postgres and ignores `DATABASE_URL`; -use `--database-url sqlite://` to export SQLite SQL. +use `--database-url mysql://` or `--database-url sqlite://` to select its SQL dialect. ```sh riverqueue migrate-get --version 3 --up > river_3.up.sql diff --git a/rust/riverqueue-cli/src/migrate.rs b/rust/riverqueue-cli/src/migrate.rs index eb3d1b6ba..b17d5d039 100644 --- a/rust/riverqueue-cli/src/migrate.rs +++ b/rust/riverqueue-cli/src/migrate.rs @@ -4,11 +4,15 @@ use std::{error::Error, time::Duration}; #[cfg(feature = "postgres")] use riverqueue::database::SchemaName; +#[cfg(feature = "mysql")] +use riverqueue_migrate::MySqlMigrator; #[cfg(feature = "postgres")] use riverqueue_migrate::PostgresMigrator; #[cfg(feature = "sqlite")] use riverqueue_migrate::SqliteMigrator; use riverqueue_migrate::{Direction, MigrateOpts, MigrateResult, Migration, ValidateResult}; +#[cfg(feature = "mysql")] +use sqlx::MySqlPool; #[cfg(feature = "postgres")] use sqlx::PgPool; #[cfg(feature = "sqlite")] @@ -28,6 +32,8 @@ struct Args { enum CommandMigrator { #[cfg(feature = "postgres")] Postgres(PostgresMigrator), + #[cfg(feature = "mysql")] + MySql(MySqlMigrator), #[cfg(feature = "sqlite")] Sqlite(SqliteMigrator), } @@ -37,6 +43,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => PostgresMigrator::all_versions(), + #[cfg(feature = "mysql")] + Self::MySql(_) => MySqlMigrator::all_versions(), #[cfg(feature = "sqlite")] Self::Sqlite(_) => SqliteMigrator::all_versions(), } @@ -46,6 +54,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.existing_versions().await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.existing_versions().await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.existing_versions().await, } @@ -59,6 +69,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.migrate(direction, opts).await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.migrate(direction, opts).await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.migrate(direction, opts).await, } @@ -71,6 +83,8 @@ impl CommandMigrator { match self { #[cfg(feature = "postgres")] Self::Postgres(migrator) => migrator.validate(target_version).await, + #[cfg(feature = "mysql")] + Self::MySql(migrator) => migrator.validate(target_version).await, #[cfg(feature = "sqlite")] Self::Sqlite(migrator) => migrator.validate(target_version).await, } @@ -87,7 +101,19 @@ pub(crate) async fn run( println!("{}", usage()); return Ok(()); }; - let migrator = if args.database_url.starts_with("sqlite:") { + let migrator = if args.database_url.starts_with("mysql:") { + #[cfg(not(feature = "mysql"))] + return Err("MySQL support requires the `mysql` feature".into()); + #[cfg(feature = "mysql")] + { + if args.schema.is_some() { + return Err("select the MySQL database in --database-url".into()); + } + CommandMigrator::MySql(MySqlMigrator::new( + MySqlPool::connect(&args.database_url).await?, + )) + } + } else if args.database_url.starts_with("sqlite:") { #[cfg(not(feature = "sqlite"))] return Err("SQLite support requires the `sqlite` feature".into()); #[cfg(feature = "sqlite")] diff --git a/rust/riverqueue-cli/src/migrate_get.rs b/rust/riverqueue-cli/src/migrate_get.rs index a78fbc5a1..65bb6239f 100644 --- a/rust/riverqueue-cli/src/migrate_get.rs +++ b/rust/riverqueue-cli/src/migrate_get.rs @@ -5,6 +5,8 @@ use std::{ io::{self, Write}, }; +#[cfg(feature = "mysql")] +use riverqueue_migrate::MySqlMigrator; #[cfg(feature = "postgres")] use riverqueue_migrate::PostgresMigrator; #[cfg(feature = "sqlite")] @@ -30,7 +32,7 @@ Options: Pass exactly one of --all or --version, and exactly one of --up or --down. Explicit versions retain their supplied order, including for down migrations. No database connection is made. DATABASE_URL is ignored; use --database-url -to select Postgres (postgres:// or postgresql://) or SQLite (sqlite://). +to select MySQL (mysql://), Postgres (postgres:// or postgresql://), or SQLite (sqlite://). Only the main migration line is bundled. Examples: @@ -127,15 +129,22 @@ fn all_versions(database_url: &str) -> Result<&'static [Migration], Box { + #[cfg(feature = "mysql")] + return Ok(MySqlMigrator::all_versions()); + #[cfg(not(feature = "mysql"))] + return Err("MySQL support requires the `mysql` feature".into()); + } Some("sqlite") => { #[cfg(feature = "sqlite")] return Ok(SqliteMigrator::all_versions()); #[cfg(not(feature = "sqlite"))] return Err("SQLite support requires the `sqlite` feature".into()); } - _ => { - Err("--database-url must use a postgres://, postgresql://, or sqlite:// prefix".into()) - } + _ => Err( + "--database-url must use a mysql://, postgres://, postgresql://, or sqlite:// prefix" + .into(), + ), } } diff --git a/rust/riverqueue-cli/tests/migrate_get.rs b/rust/riverqueue-cli/tests/migrate_get.rs index ae87052ee..6a7c437c9 100644 --- a/rust/riverqueue-cli/tests/migrate_get.rs +++ b/rust/riverqueue-cli/tests/migrate_get.rs @@ -26,6 +26,24 @@ fn help_documents_export() { } } +#[cfg(feature = "mysql")] +#[test] +fn mysql_export() { + exercise_export( + "mysql://", + riverqueue_migrate::MySqlMigrator::all_versions(), + ); +} + +#[cfg(not(feature = "mysql"))] +#[test] +fn mysql_requires_feature() { + failure( + run("mysql://", &["--all", "--up"]), + "MySQL support requires the `mysql` feature", + ); +} + #[cfg(feature = "postgres")] #[test] fn postgres_export() { @@ -221,8 +239,8 @@ fn exercise_invalid_arguments(database_url: &str) { "migration line does not exist: missing", ), ( - vec!["--all", "--up", "--database-url", "mysql://"], - "--database-url must use a postgres://, postgresql://, or sqlite:// prefix", + vec!["--all", "--up", "--database-url", "unknown://"], + "--database-url must use a mysql://, postgres://, postgresql://, or sqlite:// prefix", ), ] { failure(run(database_url, &arguments), error); diff --git a/rust/riverqueue-migrate/Cargo.toml b/rust/riverqueue-migrate/Cargo.toml index 3bfe0a1cf..8accfcf83 100644 --- a/rust/riverqueue-migrate/Cargo.toml +++ b/rust/riverqueue-migrate/Cargo.toml @@ -20,6 +20,7 @@ all-features = true [features] default = ["postgres"] postgres = ["sqlx/postgres"] +mysql = ["sqlx/mysql", "sqlx/tls-rustls"] sqlite = ["sqlx/sqlite"] [dependencies] diff --git a/rust/riverqueue-migrate/README.md b/rust/riverqueue-migrate/README.md index 6441972f6..471fcdeb2 100644 --- a/rust/riverqueue-migrate/README.md +++ b/rust/riverqueue-migrate/README.md @@ -1,9 +1,9 @@ # riverqueue-migrate -River's Postgres and SQLite migrations for Rust, identical to the ones other -River clients apply, so a client in any language can migrate a database the -others use. The `riverqueue` command from the `riverqueue-cli` crate runs the -same migrations from a shell. +River's Postgres, MySQL, and SQLite migrations for Rust, identical to the ones +other River clients apply, so a client in any language can migrate a database +the others use. The `riverqueue` command from the `riverqueue-cli` crate runs +the same migrations from a shell. Use `PostgresMigrator` for Postgres and `SqliteMigrator` for SQLite: @@ -33,3 +33,10 @@ async fn migrate(postgres: PgPool, sqlite: SqlitePool) -> Result<(), Box= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration (version); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql new file mode 100644 index 000000000..095aa1dde --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.down.sql @@ -0,0 +1,2 @@ +DROP TABLE /* TEMPLATE: schema */river_job; +DROP TABLE /* TEMPLATE: schema */river_leader; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql new file mode 100644 index 000000000..f841bf89a --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/002_initial_schema.up.sql @@ -0,0 +1,42 @@ +CREATE TABLE /* TEMPLATE: schema */river_job( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'available', + attempt INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL, + attempted_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + finalized_at DATETIME(6) NULL, + scheduled_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + priority SMALLINT NOT NULL DEFAULT 1, + args JSON NULL, + attempted_by JSON NULL, -- JSON array of strings (no native text[] in MySQL) + errors JSON NULL, -- JSON array of error objects (no native jsonb[] in MySQL) + kind VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + queue VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT 'default', + tags JSON NULL, -- JSON array of strings (no native varchar[] in MySQL) + CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL + ), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (CHAR_LENGTH(queue) > 0 AND CHAR_LENGTH(queue) < 128), + CONSTRAINT kind_length CHECK (CHAR_LENGTH(kind) > 0 AND CHAR_LENGTH(kind) < 128), + CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled')) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_job_kind ON /* TEMPLATE: schema */river_job (kind); +CREATE INDEX river_job_state_and_finalized_at_index ON /* TEMPLATE: schema */river_job (state, finalized_at); +CREATE INDEX river_job_prioritized_fetching_index ON /* TEMPLATE: schema */river_job (state, queue, priority, scheduled_at, id); + +-- MySQL has no LISTEN/NOTIFY transport, so river_job_notify is omitted. + +CREATE TABLE /* TEMPLATE: schema */river_leader ( + elected_at DATETIME(6) NOT NULL, + expires_at DATETIME(6) NOT NULL, + leader_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT leader_id_length CHECK (CHAR_LENGTH(leader_id) > 0 AND CHAR_LENGTH(leader_id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql new file mode 100644 index 000000000..93c336389 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.down.sql @@ -0,0 +1 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NULL; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql new file mode 100644 index 000000000..61b2f4303 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/003_river_job_tags_non_null.up.sql @@ -0,0 +1,2 @@ +UPDATE /* TEMPLATE: schema */river_job SET tags = JSON_ARRAY() WHERE tags IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN tags JSON NOT NULL DEFAULT (JSON_ARRAY()); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql new file mode 100644 index 000000000..891a0e813 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.down.sql @@ -0,0 +1,21 @@ +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NULL; + +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Cannot safely remove 'pending' from the CHECK constraint if rows reference +-- it, but we restore the original constraint form. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR + finalized_at IS NULL +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so no trigger changes. + +DROP TABLE /* TEMPLATE: schema */river_queue; + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name DROP DEFAULT; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql new file mode 100644 index 000000000..b544e5000 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/004_pending_and_more.up.sql @@ -0,0 +1,48 @@ +-- Make args NOT NULL with a default. +UPDATE /* TEMPLATE: schema */river_job SET args = JSON_OBJECT() WHERE args IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN args JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Make metadata NOT NULL (it already had a default). +UPDATE /* TEMPLATE: schema */river_job SET metadata = JSON_OBJECT() WHERE metadata IS NULL; +ALTER TABLE /* TEMPLATE: schema */river_job MODIFY COLUMN metadata JSON NOT NULL DEFAULT (JSON_OBJECT()); + +-- Add 'pending' to the set of valid states. States use VARCHAR with an +-- explicit constraint, so changing them does not require a table rebuild. +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT state_valid; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT state_valid CHECK ( + state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled') +); + +-- Update the finalized_at constraint to use the inverted form (matching +-- Postgres migration 004). +ALTER TABLE /* TEMPLATE: schema */river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE /* TEMPLATE: schema */river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +-- MySQL does not support triggers for LISTEN/NOTIFY, so river_job_notify +-- changes from Postgres are omitted. + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_queue ( + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Alter `river_leader` to add a default value of 'default' to `name`. +-- MySQL supports ALTER TABLE for changing defaults and constraints. +-- + +ALTER TABLE /* TEMPLATE: schema */river_leader + ALTER COLUMN name SET DEFAULT 'default'; + +ALTER TABLE /* TEMPLATE: schema */river_leader DROP CONSTRAINT name_length; +ALTER TABLE /* TEMPLATE: schema */river_leader ADD CONSTRAINT name_length CHECK (name = 'default'); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql new file mode 100644 index 000000000..5906e5567 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.down.sql @@ -0,0 +1,35 @@ +-- +-- Revert to migration table based only on `(version)`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_migration_old ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + version BIGINT NOT NULL, + CONSTRAINT version CHECK (version >= 1) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE UNIQUE INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration_old (version); + +INSERT INTO /* TEMPLATE: schema */river_migration_old + (created_at, version) +SELECT created_at, version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_old RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Drop `river_job.unique_key` and its index. +-- + +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_key; + +-- +-- Drop `river_client` and derivative. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql new file mode 100644 index 000000000..b1e83547d --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/005_migration_unique_client.up.sql @@ -0,0 +1,60 @@ +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DROP INDEX river_migration_version_idx ON /* TEMPLATE: schema */river_migration; + +CREATE TABLE /* TEMPLATE: schema */river_migration_new ( + line VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + version BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + CONSTRAINT line_length CHECK (CHAR_LENGTH(line) > 0 AND CHAR_LENGTH(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +INSERT INTO /* TEMPLATE: schema */river_migration_new + (created_at, line, version) +SELECT created_at, 'main', version +FROM /* TEMPLATE: schema */river_migration; + +DROP TABLE /* TEMPLATE: schema */river_migration; + +ALTER TABLE /* TEMPLATE: schema */river_migration_new RENAME TO /* TEMPLATE: schema */river_migration; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_key VARBINARY(255) NULL; + +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); + +-- +-- Create `river_client` and derivative. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql new file mode 100644 index 000000000..0e4bf09ff --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.down.sql @@ -0,0 +1,9 @@ +-- +-- Drop the functional unique index and unique_states column. +-- + +DROP INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job; +ALTER TABLE /* TEMPLATE: schema */river_job DROP COLUMN unique_states; + +-- Recreate the old unique index from migration 005. +CREATE UNIQUE INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql new file mode 100644 index 000000000..2f4883cce --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/006_bulk_unique.up.sql @@ -0,0 +1,45 @@ +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- MySQL 8.0+ supports functional indexes (index on an expression). We use one +-- to index `unique_key` only when the job's state matches the bitmask, which +-- is equivalent to Postgres's partial index with `river_job_state_in_bitmask`. +-- The expression evaluates to NULL when the constraint shouldn't be active, +-- and MySQL's UNIQUE allows multiple NULLs. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN unique_states SMALLINT NULL; + +-- A "functional index" (feature specific to MySQL 8.0+) over `unique_key`, +-- `unique_states`, and `state`. The expression evaluates to `unique_key` when +-- the job's current state has its corresponding bit set in `unique_states`, and +-- to NULL otherwise. MySQL's UNIQUE indexes permit multiple NULL values, so +-- rows where the constraint is inactive (NULL result) never conflict with each +-- other, while rows where it's active are checked for uniqueness on +-- `unique_key`. This is the MySQL equivalent of Postgres's partial index with +-- `WHERE ... AND river_job_state_in_bitmask(unique_states, state)`. +-- +-- Unlike Postgres's partial indexes which exclude non-matching rows from the +-- index entirely, MySQL's functional index still stores an entry for every row +-- (NULLs included), so the storage savings aren't equivalent. The uniqueness +-- semantics are the same though. +CREATE UNIQUE INDEX river_job_unique_idx ON /* TEMPLATE: schema */river_job (( + CASE WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL AND + CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + THEN unique_key + ELSE NULL + END +)); + +-- Remove the old unique index from migration 005. +DROP INDEX river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql new file mode 100644 index 000000000..1f368209e --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql @@ -0,0 +1,57 @@ +-- +-- SQL cleanup rollback. +-- + +-- +-- Add back unused tables `river_client` and `river_client_queue`. +-- + +CREATE TABLE /* TEMPLATE: schema */river_client ( + id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + paused_at DATETIME(6) NULL, + updated_at DATETIME(6) NOT NULL, + CONSTRAINT client_name_length CHECK (CHAR_LENGTH(id) > 0 AND CHAR_LENGTH(id) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE TABLE /* TEMPLATE: schema */river_client_queue ( + river_client_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + name VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + max_workers INT NOT NULL DEFAULT 0, + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + num_jobs_completed BIGINT NOT NULL DEFAULT 0, + num_jobs_running BIGINT NOT NULL DEFAULT 0, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT fk_river_client FOREIGN KEY (river_client_id) REFERENCES river_client (id) ON DELETE CASCADE, + CONSTRAINT cq_name_length CHECK (CHAR_LENGTH(name) > 0 AND CHAR_LENGTH(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +-- +-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + MODIFY COLUMN max_attempts INT NOT NULL; + +-- +-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL; + +-- +-- SQLite JSONB conversion rollback. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- Notification outbox rollback. +-- + +DROP TABLE /* TEMPLATE: schema */river_notification; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql new file mode 100644 index 000000000..82ec71552 --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql @@ -0,0 +1,44 @@ +-- +-- Notification outbox. +-- + +CREATE TABLE /* TEMPLATE: schema */river_notification ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + created_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)), + payload TEXT NOT NULL, + topic VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + CONSTRAINT topic_length CHECK (CHAR_LENGTH(topic) > 0 AND CHAR_LENGTH(topic) < 128) +) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin; + +CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at); +CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id); + +-- +-- SQLite JSONB conversion. +-- +-- No-op. MySQL stores River JSON columns in JSON columns. + +-- +-- SQL cleanup. +-- + +-- +-- Drop unused tables `river_client` and `river_client_queue`. +-- + +DROP TABLE /* TEMPLATE: schema */river_client_queue; +DROP TABLE /* TEMPLATE: schema */river_client; + +-- +-- Adds `DEFAULT 25` to `river_job.max_attempts`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_job + ALTER COLUMN max_attempts SET DEFAULT 25; + +-- +-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`. +-- + +ALTER TABLE /* TEMPLATE: schema */river_queue + MODIFY COLUMN updated_at DATETIME(6) NOT NULL DEFAULT (UTC_TIMESTAMP(6)); diff --git a/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.down.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql new file mode 100644 index 000000000..f333f931e --- /dev/null +++ b/rust/riverqueue-migrate/migrations/mysql/main/008_job_id_autoincrement.up.sql @@ -0,0 +1,2 @@ +-- No-op. MySQL already uses AUTO_INCREMENT for automatically generated job IDs. +SELECT 1; diff --git a/rust/riverqueue-migrate/src/lib.rs b/rust/riverqueue-migrate/src/lib.rs index 912431fa8..6408ebc1a 100644 --- a/rust/riverqueue-migrate/src/lib.rs +++ b/rust/riverqueue-migrate/src/lib.rs @@ -1,4 +1,4 @@ -// The README's example uses both backends, so it's the crate documentation +// The README's Postgres and SQLite example needs both features, so it's the crate documentation // (and its example runs as a doctest) when both are enabled, as on docs.rs. #![cfg_attr( all(feature = "postgres", feature = "sqlite"), @@ -6,7 +6,7 @@ )] #![cfg_attr( not(all(feature = "postgres", feature = "sqlite")), - doc = "River's Postgres and SQLite migrations for Rust. The full crate documentation, \ + doc = "River's MySQL, Postgres, and SQLite migrations for Rust. The full crate documentation, \ from the README, builds with both the `postgres` and `sqlite` features, as on \ [docs.rs](https://docs.rs/riverqueue-migrate)." )] @@ -14,8 +14,10 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -#[cfg(not(any(feature = "postgres", feature = "sqlite")))] -compile_error!("riverqueue-migrate requires at least one database feature: `postgres` or `sqlite`"); +#[cfg(not(any(feature = "postgres", feature = "sqlite", feature = "mysql")))] +compile_error!( + "riverqueue-migrate requires at least one database feature: `mysql`, `postgres`, or `sqlite`" +); use std::time::Duration; #[cfg(feature = "postgres")] @@ -25,7 +27,11 @@ use std::time::Instant; use sqlx::{PgPool, Row}; use thiserror::Error; +#[cfg(feature = "mysql")] +mod mysql; mod schema; +#[cfg(feature = "mysql")] +pub use mysql::{MYSQL_MIGRATIONS, MySqlMigrator}; #[cfg(feature = "sqlite")] mod sqlite; @@ -484,6 +490,7 @@ impl PostgresMigrator { /// Runs a migration step on its own task and waits for it, so the step /// finishes even when the caller stops waiting. +#[cfg(any(feature = "postgres", feature = "sqlite"))] async fn run_to_completion( step: impl std::future::Future> + Send + 'static, ) -> Result<(), Error> { diff --git a/rust/riverqueue-migrate/src/mysql.rs b/rust/riverqueue-migrate/src/mysql.rs new file mode 100644 index 000000000..a9d4bebe9 --- /dev/null +++ b/rust/riverqueue-migrate/src/mysql.rs @@ -0,0 +1,264 @@ +use std::time::{Duration, Instant}; + +use sqlx::{MySqlConnection, MySqlPool}; + +use crate::{ + Direction, Error, MIGRATION_LINE_MAIN, MigrateOpts, MigrateResult, MigrateVersion, Migration, + ValidateResult, select_migrations, validate_migrations, validate_target, +}; + +macro_rules! mysql_migration { + ($version:literal, $name:literal, $file:literal) => { + Migration { + down_sql: include_str!(concat!("../migrations/mysql/main/", $file, ".down.sql")), + name: $name, + up_sql: include_str!(concat!("../migrations/mysql/main/", $file, ".up.sql")), + version: $version, + } + }; +} + +/// Canonical MySQL migration bundle, shared with River's other clients. +pub const MYSQL_MIGRATIONS: [Migration; 8] = [ + mysql_migration!(1, "create_river_migration", "001_create_river_migration"), + mysql_migration!(2, "initial_schema", "002_initial_schema"), + mysql_migration!(3, "river_job_tags_non_null", "003_river_job_tags_non_null"), + mysql_migration!(4, "pending_and_more", "004_pending_and_more"), + mysql_migration!(5, "migration_unique_client", "005_migration_unique_client"), + mysql_migration!(6, "bulk_unique", "006_bulk_unique"), + mysql_migration!( + 7, + "notification_outbox_sqlite_jsonb_and_sql_cleanup", + "007_notification_outbox_sqlite_jsonb_and_sql_cleanup" + ), + mysql_migration!(8, "job_id_autoincrement", "008_job_id_autoincrement"), +]; + +/// Applies migrations to the MySQL database selected by the pool. +/// +/// The database must already exist. MySQL implicitly commits DDL: successful +/// statements remain applied after an error. A named lock serializes migrators, +/// and a version is recorded only after all its statements succeed. +#[derive(Clone, Debug)] +pub struct MySqlMigrator { + pool: MySqlPool, +} + +impl MySqlMigrator { + /// Creates a migrator for the pool's database. + #[must_use] + pub const fn new(pool: MySqlPool) -> Self { + Self { pool } + } + + /// Returns every bundled MySQL migration. + #[must_use] + pub fn all_versions() -> &'static [Migration] { + &MYSQL_MIGRATIONS + } + + /// Returns applied main-line versions in ascending order. + /// + /// # Errors + /// Returns the database error if the history cannot be read. + pub async fn existing_versions(&self) -> Result, Error> { + existing_versions(&mut *self.pool.acquire().await?).await + } + + /// Applies migrations with target, step, and dry-run controls. + /// + /// # Errors + /// Returns invalid-target errors or database errors, including failure to + /// acquire the migration lock within ten seconds. Other migration lines + /// prevent reverting version 5. + /// + /// # Cancel safety + /// Migration runs on its own task. Dropping this future lets the selected + /// migrations finish and releases the named lock before returning the + /// connection to its pool. + /// + /// # Panics + /// Resumes a panic from the migration task. + pub async fn migrate( + &self, + direction: Direction, + opts: MigrateOpts, + ) -> Result { + validate_target(&MYSQL_MIGRATIONS, opts.target_version, true)?; + let migrator = self.clone(); + match tokio::spawn(async move { migrator.migrate_locked(direction, opts).await }).await { + Ok(result) => result, + Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), + Err(error) => Err(sqlx::Error::Io(std::io::Error::other(format!( + "migration task failed: {error}" + ))) + .into()), + } + } + + /// Applies all outstanding migrations and returns their versions. + /// + /// # Errors + /// Returns the database error if a migration fails. + pub async fn migrate_up(&self) -> Result, Error> { + Ok(self + .migrate(Direction::Up, MigrateOpts::default()) + .await? + .versions + .into_iter() + .map(|version| version.version) + .collect()) + } + + /// Checks that every migration through the optional target is applied. + /// + /// # Errors + /// Returns invalid-target errors or errors reading migration history. + pub async fn validate(&self, target_version: Option) -> Result { + validate_target(&MYSQL_MIGRATIONS, target_version, false)?; + let applied = self.existing_versions().await?; + Ok(validate_migrations( + &MYSQL_MIGRATIONS, + target_version, + &applied, + )) + } + + async fn migrate_locked( + &self, + direction: Direction, + opts: MigrateOpts, + ) -> Result { + let mut connection = self.pool.acquire().await?; + let acquired: Option = + sqlx::query_scalar("SELECT GET_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)), 10)") + .fetch_one(&mut *connection) + .await?; + if acquired != Some(1) { + return Err(sqlx::Error::Protocol( + "could not acquire MySQL migration lock; select a database in the connection URL" + .into(), + ) + .into()); + } + // A task abort or panic closes the connection, releasing its named lock. + connection.close_on_drop(); + let result = migrate_on(&mut connection, direction, opts).await; + let released = sqlx::query("SELECT RELEASE_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)))") + .execute(&mut *connection) + .await; + match result { + Ok(result) => { + released?; + Ok(result) + } + Err(error) => Err(error), + } + } +} + +async fn existing_versions(connection: &mut MySqlConnection) -> Result, Error> { + let exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = 'river_migration')").fetch_one(&mut *connection).await?; + if !exists { + return Ok(Vec::new()); + } + let has_line: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = 'river_migration' AND column_name = 'line')").fetch_one(&mut *connection).await?; + Ok(if has_line { + sqlx::query_scalar( + "SELECT CAST(version AS SIGNED) FROM river_migration WHERE line = ? ORDER BY version", + ) + .bind(MIGRATION_LINE_MAIN) + .fetch_all(connection) + .await? + } else { + sqlx::query_scalar("SELECT CAST(version AS SIGNED) FROM river_migration ORDER BY version") + .fetch_all(connection) + .await? + }) +} + +async fn migrate_on( + connection: &mut MySqlConnection, + direction: Direction, + opts: MigrateOpts, +) -> Result { + let applied = existing_versions(connection).await?; + let selected = select_migrations(&MYSQL_MIGRATIONS, direction, opts, &applied)?; + let mut versions = Vec::with_capacity(selected.len()); + for migration in selected { + let sql = match direction { + Direction::Down => migration.down_sql, + Direction::Up => migration.up_sql, + } + .replace("/* TEMPLATE: schema */", ""); + let mut duration = Duration::ZERO; + if !opts.dry_run { + let start = Instant::now(); + apply(connection, direction, migration, &sql).await?; + duration = start.elapsed(); + } + versions.push(MigrateVersion { + duration, + name: migration.name, + sql, + version: migration.version, + }); + } + Ok(MigrateResult { + direction, + versions, + }) +} + +async fn apply( + connection: &mut MySqlConnection, + direction: Direction, + migration: Migration, + sql: &str, +) -> Result<(), Error> { + if direction == Direction::Down && migration.version == 5 { + let other_lines: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM river_migration WHERE line <> ?)") + .bind(MIGRATION_LINE_MAIN) + .fetch_one(&mut *connection) + .await?; + if other_lines { + return Err(Error::OtherMigrationLines { + version: migration.version, + }); + } + } + sqlx::raw_sql(sqlx::AssertSqlSafe(sql)) + .execute(&mut *connection) + .await?; + match direction { + Direction::Down if migration.version == 1 => {} + Direction::Down if migration.version <= 5 => { + sqlx::query("DELETE FROM river_migration WHERE version = ?") + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Down => { + sqlx::query("DELETE FROM river_migration WHERE line = ? AND version = ?") + .bind(MIGRATION_LINE_MAIN) + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Up if migration.version >= 5 => { + sqlx::query("INSERT INTO river_migration (line, version) VALUES (?, ?)") + .bind(MIGRATION_LINE_MAIN) + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + Direction::Up => { + sqlx::query("INSERT INTO river_migration (version) VALUES (?)") + .bind(migration.version) + .execute(&mut *connection) + .await?; + } + } + Ok(()) +} diff --git a/rust/riverqueue-test/Cargo.toml b/rust/riverqueue-test/Cargo.toml index 24e3a29c5..a65d36da2 100644 --- a/rust/riverqueue-test/Cargo.toml +++ b/rust/riverqueue-test/Cargo.toml @@ -20,6 +20,7 @@ all-features = true [features] default = ["postgres"] postgres = ["riverqueue/postgres"] +mysql = ["riverqueue/mysql"] sqlite = ["riverqueue/sqlite"] [dependencies] diff --git a/rust/riverqueue/Cargo.toml b/rust/riverqueue/Cargo.toml index fa9d5b65c..341cd27fc 100644 --- a/rust/riverqueue/Cargo.toml +++ b/rust/riverqueue/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true homepage.workspace = true # Only what builds and documents the crate ships; tests and their fixtures stay # in the repository. -include = ["src/**", "docs/**", "examples/**", "LICENSE", "README.md"] +include = ["src/**", "!src/**/tests/**", "docs/**", "examples/**", "LICENSE", "README.md"] [package.metadata.docs.rs] all-features = true @@ -24,6 +24,7 @@ sqlite = ["riverqueue-migrate/sqlite", "sqlx/sqlite"] # Resolves IANA zone names such as `America/New_York` in cron `CRON_TZ=` # prefixes from a bundled time zone database. tzdb = ["jiff/tzdb-bundle-always"] +mysql = ["riverqueue-migrate/mysql", "sqlx/mysql", "sqlx/tls-rustls", "dep:sqlx-core"] [dependencies] async-trait.workspace = true @@ -38,6 +39,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlx = { workspace = true, features = ["derive", "json"] } +sqlx-core = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/rust/riverqueue/README.md b/rust/riverqueue/README.md index 256ead0b1..0fca69b6b 100644 --- a/rust/riverqueue/README.md +++ b/rust/riverqueue/README.md @@ -1,7 +1,7 @@ # riverqueue `riverqueue` is the Rust and Tokio client for [River](https://riverqueue.com), -a fast and reliable background job system backed by Postgres or SQLite. It +a fast and reliable background job system backed by Postgres, MySQL, or SQLite. It shares River's database schema and job protocol with River clients in other languages, so services written in different languages can insert and work jobs in the same database. @@ -24,6 +24,7 @@ Rust version is 1.95. | Feature | Default | Enables | |---|---|---| +| `mysql` | no | MySQL 8.0 or newer through SQLx | | `postgres` | yes | Postgres through SQLx | | `sqlite` | no | SQLite 3.45 or newer through SQLx | | `tzdb` | no | IANA zone names such as `America/New_York` in cron `CRON_TZ=` and `TZ=` prefixes | @@ -42,6 +43,40 @@ implementation for SQLx; enable one of SQLx's TLS features in your own SQLx dependency if your database connections use TLS. For upgrades from Chrono, see the [migration guide](https://docs.rs/riverqueue/latest/riverqueue/guide/migrating_from_chrono/index.html). +## MySQL + +Enable the `mysql` feature, and pass a SQLx `MySqlPool` to the client: + +```toml +riverqueue = { version = "0.3.0", default-features = false, features = ["mysql"] } +``` + +The MySQL feature includes SQLx's Rustls transport for password authentication +over TLS. + +```rust,no_run +# #[cfg(feature = "mysql")] +# async fn example() -> Result<(), Box> { +use riverqueue::{Client, migrate::MySqlMigrator, sqlx::MySqlPool}; + +let pool = MySqlPool::connect("mysql://localhost/app").await?; +MySqlMigrator::new(pool.clone()).migrate_up().await?; +let client = Client::builder(pool).build()?; +# Ok(()) +# } +``` + +Create the database before migrating. Jobs, workers, queues, and `.tx(...)` +requests use the same APIs as the other backends. MySQL clients poll River's +transactional notification outbox for insertions, queue changes, cancellation, +and leadership requests. River's own transactions use READ COMMITTED isolation; +caller-owned transactions retain their configured isolation. Use +`riverqueue::database::begin_mysql_write` for a cancellation-safe begin at +READ COMMITTED. + +MySQL implicitly commits DDL, so a failed migration may leave successful +statements applied. Migration history advances only after all statements in +that migration succeed. ## Quick start @@ -337,7 +372,7 @@ async fn sqlite_client() -> Result> { - [`queue`] and [`query`]: queue records and job list filters and cursors. - [`periodic`]: schedules and runtime periodic job registration. - [`extension`]: hooks, middleware, policies, and metrics. -- [`database`]: Postgres and SQLite database options, and the transactions +- [`database`]: MySQL, Postgres, and SQLite database options, and the transactions River's `.tx` methods accept. - [`error`]: structured errors that keep their sources. - [`protocol`]: wire values such as notification topics and unique keys, for diff --git a/rust/riverqueue/src/__private.rs b/rust/riverqueue/src/__private.rs index 8f329b348..2490afa30 100644 --- a/rust/riverqueue/src/__private.rs +++ b/rust/riverqueue/src/__private.rs @@ -15,6 +15,8 @@ use serde_json::{Map, Value, value::RawValue}; #[cfg(feature = "postgres")] use crate::database::SchemaName; use crate::{AttemptError, InsertResult, Job, JobRow, JobState}; +#[cfg(feature = "mysql")] +use sqlx::{MySqlConnection, MySqlPool}; #[cfg(feature = "postgres")] use sqlx::{PgConnection, PgPool}; #[cfg(feature = "sqlite")] @@ -23,6 +25,8 @@ use tokio_util::sync::CancellationToken; pub use crate::client::{ExtensionClient, PreparedInsertRequest, RawInsertRequest, WeakClient}; pub use crate::database::erased::{Database, ErasedExecutor, ErasedTransaction}; +#[cfg(feature = "mysql")] +pub use crate::database::mysql::{Connection as MySqlConnectionGuard, Transaction as MySqlTransactionGuard}; pub use crate::pilot::{ JobDeleteBeforeContext, JobDeleteBeforeNext, JobDeleteContext, JobDeleteManyContext, JobDeleteManyNext, JobDeleteNext, PilotDatabase, PilotProducer, PilotTransaction, @@ -173,6 +177,15 @@ pub fn attempt_errors_from_json(json: &str) -> Result, serde_j AttemptError::from_json_array_lenient(json) } +/// Encodes a UTC timestamp in River's canonical MySQL wire format. +/// +/// This keeps companion crates aligned with River and Go's +/// microsecond-precision, timezone-free MySQL representation. +#[cfg(feature = "mysql")] +#[must_use] +pub fn mysql_timestamp(time: Timestamp) -> String { + crate::database::mysql::mysql_time(time) +} /// Encodes a UTC timestamp in River's canonical SQLite wire format. /// /// This keeps companion crates aligned with River and Go's @@ -353,6 +366,17 @@ pub async fn notify_many( .await?; Ok(()) } + #[cfg(feature = "mysql")] + (DatabaseConnection::MySql(connection), DatabaseConfig::MySql) => { + let mut query = sqlx::QueryBuilder::::new( + "INSERT INTO river_notification (payload, topic) ", + ); + query.push_values(payloads, |mut row, payload| { + row.push_bind(payload).push_bind(topic.as_str()); + }); + query.build().execute(connection).await?; + Ok(()) + } #[cfg(feature = "sqlite")] (DatabaseConnection::Sqlite(connection), DatabaseConfig::Sqlite) => { let mut query = sqlx::QueryBuilder::::new( @@ -430,6 +454,10 @@ pub async fn delete_finalized_jobs( (DatabaseConnection::Postgres(connection), DatabaseConfig::Postgres { schema }) => Ok( crate::maintenance::postgres_delete_finalized_jobs(connection, schema, params).await?, ), + #[cfg(feature = "mysql")] + (DatabaseConnection::MySql(connection), DatabaseConfig::MySql) => { + Ok(crate::maintenance::mysql_delete_finalized_jobs(connection, params).await?) + } #[cfg(feature = "sqlite")] (DatabaseConnection::Sqlite(connection), DatabaseConfig::Sqlite) => { Ok(crate::maintenance::sqlite_delete_finalized_jobs(connection, params).await?) @@ -457,6 +485,9 @@ pub enum DatabaseConfig { /// Postgres backend configuration. #[cfg(feature = "postgres")] Postgres { schema: SchemaName }, + /// MySQL backend configuration. + #[cfg(feature = "mysql")] + MySql, /// SQLite backend configuration. #[cfg(feature = "sqlite")] Sqlite, @@ -469,6 +500,8 @@ impl DatabaseConfig { match self { #[cfg(feature = "postgres")] Self::Postgres { .. } => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite => DatabaseKind::Sqlite, } @@ -480,6 +513,8 @@ impl DatabaseConfig { pub const fn postgres_schema(&self) -> Option<&SchemaName> { match self { Self::Postgres { schema } => Some(schema), + #[cfg(feature = "mysql")] + Self::MySql => None, #[cfg(feature = "sqlite")] Self::Sqlite => None, } @@ -493,6 +528,9 @@ pub enum DatabaseConnection<'connection> { /// Postgres transaction connection. #[cfg(feature = "postgres")] Postgres(&'connection mut PgConnection), + /// MySQL transaction connection. + #[cfg(feature = "mysql")] + MySql(&'connection mut MySqlConnection), /// SQLite transaction connection. #[cfg(feature = "sqlite")] Sqlite(&'connection mut SqliteConnection), @@ -505,6 +543,8 @@ impl<'connection> DatabaseConnection<'connection> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -517,6 +557,8 @@ impl<'connection> DatabaseConnection<'connection> { match self { #[cfg(feature = "postgres")] Self::Postgres(connection) => DatabaseConnection::Postgres(connection), + #[cfg(feature = "mysql")] + Self::MySql(connection) => DatabaseConnection::MySql(connection), #[cfg(feature = "sqlite")] Self::Sqlite(connection) => DatabaseConnection::Sqlite(connection), } @@ -528,6 +570,8 @@ impl<'connection> DatabaseConnection<'connection> { pub fn into_postgres(self) -> Option<&'connection mut PgConnection> { match self { Self::Postgres(connection) => Some(connection), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } @@ -535,12 +579,24 @@ impl<'connection> DatabaseConnection<'connection> { /// Returns the SQLite connection, if selected. #[must_use] + #[cfg(feature = "mysql")] + pub fn into_mysql(self) -> Option<&'connection mut MySqlConnection> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(connection) => Some(connection), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } #[cfg(feature = "sqlite")] pub fn into_sqlite(self) -> Option<&'connection mut SqliteConnection> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(connection) => Some(connection), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } } @@ -562,6 +618,9 @@ pub enum DatabasePool { /// Postgres pool. #[cfg(feature = "postgres")] Postgres(PgPool), + /// MySQL pool. + #[cfg(feature = "mysql")] + MySql(MySqlPool), /// SQLite pool. #[cfg(feature = "sqlite")] Sqlite(SqlitePool), @@ -574,6 +633,8 @@ impl DatabasePool { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -585,6 +646,8 @@ impl DatabasePool { pub const fn postgres(&self) -> Option<&PgPool> { match self { Self::Postgres(pool) => Some(pool), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } @@ -592,12 +655,24 @@ impl DatabasePool { /// Returns the caller-owned SQLite pool, if selected. #[must_use] + #[cfg(feature = "mysql")] + pub const fn mysql(&self) -> Option<&MySqlPool> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(pool) => Some(pool), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } #[cfg(feature = "sqlite")] pub const fn sqlite(&self) -> Option<&SqlitePool> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(pool) => Some(pool), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } } @@ -1208,11 +1283,25 @@ pub fn claimed_postgres_job(row: &sqlx::postgres::PgRow) -> ClaimedJob { ClaimedJob::from_decoded(crate::client::decode_job_row(row)) } +/// Columns River selects to decode a MySQL job row, for use with +/// [`decode_mysql_job_row`]. +#[cfg(feature = "mysql")] +pub const MYSQL_JOB_COLUMNS: &str = crate::database::mysql::JOB_COLUMNS; /// Columns River selects to decode a SQLite job row, for use with /// [`decode_sqlite_job_row`]. #[cfg(feature = "sqlite")] pub const SQLITE_JOB_COLUMNS: &str = crate::database::sqlite::JOB_COLUMNS; +/// Decodes a row selected with [`SQLITE_JOB_COLUMNS`] exactly as River +/// decodes its own rows. +/// +/// # Errors +/// +/// Returns an error when the row can't be decoded. +#[cfg(feature = "mysql")] +pub fn decode_mysql_job_row(row: &sqlx::mysql::MySqlRow) -> Result { + crate::database::mysql::decode_job_row(row).map_err(|undecodable| undecodable.error.into()) +} /// Decodes a row selected with [`SQLITE_JOB_COLUMNS`] exactly as River /// decodes its own rows. /// @@ -1224,6 +1313,13 @@ pub fn decode_sqlite_job_row(row: &sqlx::sqlite::SqliteRow) -> Result ClaimedJob { + ClaimedJob::from_decoded(crate::database::mysql::decode_job_row(row)) +} /// Decodes a claimed row selected with [`SQLITE_JOB_COLUMNS`] as far as /// River can, for [`PilotProducer::claim`]. #[cfg(feature = "sqlite")] diff --git a/rust/riverqueue/src/client/completer.rs b/rust/riverqueue/src/client/completer.rs index 93e7700b3..242399cb6 100644 --- a/rust/riverqueue/src/client/completer.rs +++ b/rust/riverqueue/src/client/completer.rs @@ -28,6 +28,8 @@ use tracing::{debug, error}; use crate::__private::{DatabaseConnection as PilotDatabaseConnection, JobSetStateParams}; use crate::client::attempts::remove_running_attempt; use crate::client::backoff::exponential_backoff; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::client::{ClientInner, RuntimeNotification, peers, tolerant_row}; @@ -144,6 +146,8 @@ impl CompletionBatcher { let backend_concurrency = match inner.database.kind() { #[cfg(feature = "postgres")] DatabaseKind::Postgres => COMPLETION_POSTGRES_CONCURRENCY, + #[cfg(feature = "mysql")] + DatabaseKind::MySql => 1, #[cfg(feature = "sqlite")] DatabaseKind::Sqlite => 1, }; @@ -485,8 +489,8 @@ async fn persist_with_retries( } /// Wakes producers for jobs that client shutdown returned to `available`, so -/// another client picks them up without waiting for its next poll. SQLite -/// writes its durable wakeup inside the completion transaction instead. +/// another client picks them up without waiting for its next poll. MySQL and +/// SQLite write their durable wakeups inside the completion transaction. #[cfg_attr( not(feature = "postgres"), allow(clippy::unused_async, reason = "only Postgres sends a notification") @@ -511,7 +515,9 @@ async fn notify_interrupted_jobs(inner: &ClientInner, batch: &[CompletionUpdate] .send(RuntimeNotification::Insert((*queue).to_owned())); } match inner.database.pool() { - // SQLite clients wrote these notifications with the completion. + // Outbox clients wrote these notifications with the completion. + #[cfg(feature = "mysql")] + DatabasePool::MySql(_) => {} #[cfg(feature = "sqlite")] DatabasePool::Sqlite(_) => {} #[cfg(feature = "postgres")] @@ -558,6 +564,73 @@ pub(super) async fn persist_completion_batch( batch: &[CompletionUpdate], ) -> Result, Error> { match inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let mut rows = Vec::with_capacity(batch.len()); + let now = Timestamp::now(); + for update in batch { + let row = crate::database::mysql::complete_decoded( + &mut transaction, + &crate::database::mysql::CompleteJob { + attempt: update.attempt, + error: update.error.as_ref(), + finalized_at: update.finalized_at, + id: update.job_id, + metadata_updates: (!update.metadata.is_empty()).then_some(&update.metadata), + now, + scheduled_at: update.scheduled_at, + state: update.state, + }, + ) + .await + .map_err(mysql_backend_error)?; + let row = match row { + Some(row) => Some(row), + None => crate::database::mysql::merge_metadata_if_not_running( + &mut transaction, + update.job_id, + &update.metadata, + ) + .await + .map_err(mysql_backend_error)?, + }; + rows.extend(row.and_then(tolerant_row)); + } + let interrupted_queues = rows + .iter() + .filter(|row| row.state == JobState::Available) + .filter(|row| { + batch.iter().any(|update| { + update.job_id == row.id && update.event_kind == JobEventKind::Interrupted + }) + }) + .map(|row| row.queue.as_str()) + .collect::>(); + for queue in interrupted_queues { + let payload = serde_json::json!({ "queue": queue }).to_string(); + crate::database::mysql::notification_insert( + &mut transaction, + &[crate::database::mysql::NotificationInput { + payload: &payload, + topic: crate::NOTIFICATION_TOPIC_INSERT, + }], + ) + .await + .map_err(mysql_backend_error)?; + } + if inner.pilot.intercepts_job_set_state() { + after_jobs_set_state( + inner, + PilotDatabaseConnection::MySql(&mut transaction), + &batch.iter().map(|update| update.job_id).collect::>(), + &rows, + ) + .await?; + } + transaction.commit().await?; + Ok(rows) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; diff --git a/rust/riverqueue/src/client/insert.rs b/rust/riverqueue/src/client/insert.rs index 1e87f0e46..cdfc42fbc 100644 --- a/rust/riverqueue/src/client/insert.rs +++ b/rust/riverqueue/src/client/insert.rs @@ -33,6 +33,8 @@ use crate::__private::{ DatabaseConnection as PilotDatabaseConnection, JobInsertParams as PilotJobInsertParams, }; use crate::client::RuntimeNotification; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; use crate::client::request::Target; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; @@ -594,6 +596,19 @@ impl Client { transaction.commit().await?; inserted } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let inserted = self + .insert_and_finish( + PilotDatabaseConnection::MySql(&mut transaction), + jobs, + finish, + ) + .await?; + transaction.commit().await?; + inserted + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -812,6 +827,23 @@ impl Client { .execute(connection) .await?; } + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => { + let payloads = queues + .into_iter() + .map(crate::protocol::insert_notification_payload) + .collect::>(); + let notifications = payloads + .iter() + .map(|payload| crate::database::mysql::NotificationInput { + payload, + topic: crate::protocol::NOTIFICATION_TOPIC_INSERT, + }) + .collect::>(); + crate::database::mysql::notification_insert(connection, ¬ifications) + .await + .map_err(mysql_backend_error)?; + } #[cfg(feature = "sqlite")] PilotDatabaseConnection::Sqlite(connection) => { let payloads = queues @@ -839,18 +871,19 @@ impl Client { connection: PilotDatabaseConnection<'_>, job: InsertContext, ) -> Result { - let InsertContext { - encoded_args, - kind, - opts, - state, - created_at, - unique_key, - unique_states, - } = job; match connection { #[cfg(feature = "postgres")] PilotDatabaseConnection::Postgres(connection) => { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + let table = self.inner.schema.qualify("river_job"); let state_type = self.inner.schema.qualify("river_job_state"); let state_function = self.inner.schema.qualify("river_job_state_in_bitmask"); @@ -902,46 +935,112 @@ impl Client { let duplicate = nonce.map_or(duplicate, |nonce| lacks_nonce(&row, &nonce)); Ok(InsertedJob::new(row, duplicate)) } + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => insert_mysql_row(connection, job).await, #[cfg(feature = "sqlite")] - PilotDatabaseConnection::Sqlite(connection) => { - // Like Go's SQLite driver, every inserted row carries a nonce, - // and times left unset are filled in by SQLite's own clock. - let nonce = unique_insert_nonce(); - let inserted = crate::database::sqlite::insert( - connection, - &crate::database::sqlite::InsertJob { - attempt: 0, - attempted_at: None, - attempted_by: &[], - created_at, - encoded_args: &encoded_args, - errors: &[], - finalized_at: None, - id: None, - kind: &kind, - max_attempts: opts.max_attempts, - metadata: &opts.metadata, - priority: opts.priority, - queue: &opts.queue, - scheduled_at: opts.scheduled_at, - state, - tags: &opts.tags, - unique_key: unique_key.as_deref(), - unique_nonce: Some(&nonce), - unique_states, - }, - ) - .await - .map_err(sqlite_backend_error)?; - Ok(InsertedJob::new( - inserted.job, - inserted.unique_skipped_as_duplicate, - )) - } + PilotDatabaseConnection::Sqlite(connection) => insert_sqlite_row(connection, job).await, } } } +#[cfg(feature = "mysql")] +async fn insert_mysql_row( + connection: &mut sqlx::MySqlConnection, + job: InsertContext, +) -> Result { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + // Like Go's MySQL driver, every inserted row carries a nonce, + // and times left unset are filled in by MySQL's own clock. + let nonce = unique_insert_nonce(); + let inserted = crate::database::mysql::insert( + connection, + &crate::database::mysql::InsertJob { + attempt: 0, + attempted_at: None, + attempted_by: &[], + created_at, + encoded_args: &encoded_args, + errors: &[], + finalized_at: None, + id: None, + kind: &kind, + max_attempts: opts.max_attempts, + metadata: &opts.metadata, + priority: opts.priority, + queue: &opts.queue, + scheduled_at: opts.scheduled_at, + state, + tags: &opts.tags, + unique_key: unique_key.as_deref(), + unique_nonce: Some(&nonce), + unique_states, + }, + ) + .await + .map_err(mysql_backend_error)?; + Ok(InsertedJob::new( + inserted.job, + inserted.unique_skipped_as_duplicate, + )) +} + +#[cfg(feature = "sqlite")] +async fn insert_sqlite_row( + connection: &mut sqlx::SqliteConnection, + job: InsertContext, +) -> Result { + let InsertContext { + encoded_args, + kind, + opts, + state, + created_at, + unique_key, + unique_states, + } = job; + // Like Go's SQLite driver, every inserted row carries a nonce, + // and times left unset are filled in by SQLite's own clock. + let nonce = unique_insert_nonce(); + let inserted = crate::database::sqlite::insert( + connection, + &crate::database::sqlite::InsertJob { + attempt: 0, + attempted_at: None, + attempted_by: &[], + created_at, + encoded_args: &encoded_args, + errors: &[], + finalized_at: None, + id: None, + kind: &kind, + max_attempts: opts.max_attempts, + metadata: &opts.metadata, + priority: opts.priority, + queue: &opts.queue, + scheduled_at: opts.scheduled_at, + state, + tags: &opts.tags, + unique_key: unique_key.as_deref(), + unique_nonce: Some(&nonce), + unique_states, + }, + ) + .await + .map_err(sqlite_backend_error)?; + Ok(InsertedJob::new( + inserted.job, + inserted.unique_skipped_as_duplicate, + )) +} + /// Exposes the mutable fields of each job to an extension's insert hook. fn extension_insert_params(jobs: &mut [InsertContext]) -> Vec> { jobs.iter_mut() diff --git a/rust/riverqueue/src/client/jobs.rs b/rust/riverqueue/src/client/jobs.rs index ac8482870..21aef3c9c 100644 --- a/rust/riverqueue/src/client/jobs.rs +++ b/rust/riverqueue/src/client/jobs.rs @@ -314,7 +314,9 @@ impl JobGetRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session.storage(inner).job_get(self.id).await + let result = session.storage(inner).job_get(self.id).await; + session.commit().await?; + result } } @@ -329,6 +331,7 @@ impl JobListRequest<'_> { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; let jobs = session.storage(inner).job_list(&self.params).await?; + session.commit().await?; let last_cursor = jobs .last() .map(|job| JobListCursor::after_job(job, &self.params)); @@ -361,11 +364,13 @@ request_type! { impl JobUpdateRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; - let mut session = self.target.session(inner, Access::Autocommit).await?; - session + let mut session = self.target.session(inner, Access::Transaction).await?; + let row = session .storage(inner) .job_update(self.id, self.params) - .await + .await?; + session.commit().await?; + Ok(row) } } diff --git a/rust/riverqueue/src/client/mod.rs b/rust/riverqueue/src/client/mod.rs index 8bbcbfe9c..dd79cdc6e 100644 --- a/rust/riverqueue/src/client/mod.rs +++ b/rust/riverqueue/src/client/mod.rs @@ -36,7 +36,7 @@ pub use self::queues::{ QueueGetRequest, QueueListRequest, QueuePauseRequest, QueueResumeRequest, QueueUpdateRequest, Queues, }; -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] pub(crate) use self::record::FieldErrors; pub(crate) use self::record::{DecodedJob, UndecodableJob, tolerant_row}; #[cfg(feature = "postgres")] @@ -162,6 +162,10 @@ pub(crate) struct ClientInner { pub(crate) workers: Workers, } +#[cfg(feature = "mysql")] +fn mysql_backend_error(error: crate::database::mysql::BackendError) -> Error { + Error::Database(error.into()) +} #[cfg(feature = "sqlite")] fn sqlite_backend_error(error: crate::database::sqlite::BackendError) -> Error { Error::Database(error.into()) @@ -202,7 +206,7 @@ impl ClientInner { } /// Whether this client hears committed notifications through a notifier, - /// a Postgres listener or SQLite outbox poller, like River Go's client + /// a Postgres listener or MySQL/SQLite outbox poller, like River Go's client /// notifier. A poll-only client has none, and neither does a client of a /// Postgres server without `LISTEN`/`NOTIFY` once that's detected. pub(crate) fn has_notifier(&self) -> bool { @@ -224,6 +228,8 @@ impl ClientInner { #[cfg(feature = "postgres")] schema: self.schema.clone(), }, + #[cfg(feature = "mysql")] + DatabasePool::MySql(_) => PilotDatabaseConfig::MySql, #[cfg(feature = "sqlite")] DatabasePool::Sqlite(_) => PilotDatabaseConfig::Sqlite, } @@ -241,6 +247,8 @@ impl ClientInner { match self.database.pool() { #[cfg(feature = "postgres")] DatabasePool::Postgres(pool) => PilotDatabasePool::Postgres(pool.clone()), + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => PilotDatabasePool::MySql(pool.clone()), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => PilotDatabasePool::Sqlite(pool.clone()), } diff --git a/rust/riverqueue/src/client/notifier.rs b/rust/riverqueue/src/client/notifier.rs index be9aa976e..44173e9ba 100644 --- a/rust/riverqueue/src/client/notifier.rs +++ b/rust/riverqueue/src/client/notifier.rs @@ -24,6 +24,8 @@ use crate::Error; use crate::client::ClientInner; use crate::client::attempts::signal_running_attempt; use crate::client::backoff::exponential_backoff; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::maintenance::LeadershipWakeup; @@ -350,6 +352,103 @@ async fn listen_until_error( } } +/// Polls MySQL's notification outbox until cancelled. +/// +/// Like River Go's MySQL listener, the first successful poll connects after +/// the outbox's current maximum ID and subscribes to River's topics, so +/// earlier rows are never delivered. Poll failures are logged and retried +/// with River's service backoff. The cursor is kept, so no durable +/// notification is skipped. +#[cfg(feature = "mysql")] +pub(super) async fn run_mysql_notifications( + inner: Arc, + pool: sqlx::MySqlPool, + cancel: CancellationToken, + queue_notifications: broadcast::Sender, + ready: ReadySlot, +) -> Result<(), Error> { + let mut attempt = 0; + let mut listener = crate::database::mysql::NotificationListener::default(); + let mut notification_tick = + tokio::time::interval(crate::database::mysql::DEFAULT_NOTIFICATION_POLL_INTERVAL); + notification_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + () = cancel.cancelled() => return Ok(()), + _ = notification_tick.tick() => {} + } + // Like the rest of a poll, waiting for a connection during an + // outage must not delay a stop, as Go's cancellable context doesn't. + let connection = tokio::select! { + biased; + () = cancel.cancelled() => return Ok(()), + connection = pool.acquire() => connection, + }; + let polled = async { + let mut connection = connection?; + let receive = async { + if !listener.is_connected() { + listener + .connect(&mut connection) + .await + .map_err(mysql_backend_error)?; + } + for topic in [ + crate::NOTIFICATION_TOPIC_CONTROL, + crate::NOTIFICATION_TOPIC_INSERT, + crate::NOTIFICATION_TOPIC_LEADERSHIP, + ] { + listener + .listen(&mut connection, topic) + .await + .map_err(mysql_backend_error)?; + } + report_ready(&ready); + while let Some(notification) = listener + .next(&mut connection) + .await + .map_err(mysql_backend_error)? + { + dispatch_notification( + &inner, + &queue_notifications, + ¬ification.topic, + ¬ification.payload, + ); + } + Ok::<_, Error>(()) + }; + tokio::select! { + biased; + () = cancel.cancelled() => { + // Discard a connection with an outstanding locking read so + // neither shutdown nor the next borrower waits for its result. + drop(connection.detach()); + Ok(()) + } + result = receive => result, + } + } + .await; + if let Err(poll_error) = polled { + attempt += 1; + let sleep = exponential_backoff(attempt); + error!( + attempt, + error = %crate::error::Chain(&poll_error), + sleep_duration = ?sleep, + "River notification poll failed (will retry after backoff); producers keep polling" + ); + tokio::select! { + () = cancel.cancelled() => return Ok(()), + () = tokio::time::sleep(sleep) => {} + } + continue; + } + attempt = 0; + } +} /// Polls SQLite's notification outbox until cancelled. /// /// Like River Go's SQLite listener, the first successful poll connects after diff --git a/rust/riverqueue/src/client/producer.rs b/rust/riverqueue/src/client/producer.rs index 7cdc52029..8bc928a92 100644 --- a/rust/riverqueue/src/client/producer.rs +++ b/rust/riverqueue/src/client/producer.rs @@ -20,6 +20,8 @@ use crate::client::attempts::{FetchRegistrationGuard, register_running_attempt}; use crate::client::backoff::{RestartBackoff, exponential_backoff}; use crate::client::completer::CompletionUpdate; use crate::client::executor::{AbortOnDrop, execute_job}; +#[cfg(feature = "mysql")] +use crate::client::mysql_backend_error; #[cfg(feature = "sqlite")] use crate::client::sqlite_backend_error; use crate::client::{ @@ -1041,6 +1043,8 @@ async fn fetch_available( let use_parallel_fetch = match inner.database.kind() { #[cfg(feature = "postgres")] DatabaseKind::Postgres => true, + #[cfg(feature = "mysql")] + DatabaseKind::MySql => true, #[cfg(feature = "sqlite")] DatabaseKind::Sqlite => false, }; @@ -1139,6 +1143,20 @@ pub(crate) async fn standard_claim( .iter() .map(decode_job_row) .collect()), + #[cfg(feature = "mysql")] + PilotDatabaseConnection::MySql(connection) => { + let params = crate::database::mysql::ClaimJobs { + client_id: &inner.id, + kinds: inner.fetch_kinds.as_deref(), + limit, + max_attempted_by: ATTEMPTED_BY_MAX, + now: Timestamp::now(), + queue, + }; + crate::database::mysql::claim(connection, ¶ms) + .await + .map_err(mysql_backend_error) + } #[cfg(feature = "sqlite")] PilotDatabaseConnection::Sqlite(connection) => { let params = crate::database::sqlite::ClaimJobs { @@ -1204,6 +1222,24 @@ pub(super) async fn fetch_jobs( ) .await? } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let Some(transaction) = + unless_cancelled(cancel, crate::database::mysql::Transaction::begin(pool)).await + else { + return Ok(FetchedJobs::default()); + }; + let mut transaction = transaction?; + let rows = standard_claim( + inner, + PilotDatabaseConnection::MySql(&mut transaction), + queue, + maximum, + ) + .await?; + transaction.commit().await?; + rows + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let Some(connection) = unless_cancelled(cancel, pool.acquire()).await else { diff --git a/rust/riverqueue/src/client/queues.rs b/rust/riverqueue/src/client/queues.rs index 16dc4e217..84b9e104c 100644 --- a/rust/riverqueue/src/client/queues.rs +++ b/rust/riverqueue/src/client/queues.rs @@ -163,11 +163,9 @@ impl QueueGetRequest<'_> { async fn run(self) -> Result { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session - .storage(inner) - .queue_get(&self.name) - .await? - .ok_or_else(|| Error::NotFound(crate::Record::Queue(self.name.clone()))) + let queue = session.storage(inner).queue_get(&self.name).await?; + session.commit().await?; + queue.ok_or_else(|| Error::NotFound(crate::Record::Queue(self.name.clone()))) } } @@ -181,7 +179,9 @@ impl QueueListRequest<'_> { async fn run(self) -> Result, Error> { let inner = &self.client.inner; let mut session = self.target.session(inner, Access::Autocommit).await?; - session.storage(inner).queue_list(&self.params).await + let queues = session.storage(inner).queue_list(&self.params).await?; + session.commit().await?; + Ok(queues) } } diff --git a/rust/riverqueue/src/client/run.rs b/rust/riverqueue/src/client/run.rs index f8c42bac0..db3f264f0 100644 --- a/rust/riverqueue/src/client/run.rs +++ b/rust/riverqueue/src/client/run.rs @@ -13,6 +13,8 @@ use tracing::{debug, error, warn}; use crate::client::ClientInner; use crate::client::backoff::RestartBackoff; use crate::client::completer::run_completion_batcher; +#[cfg(feature = "mysql")] +use crate::client::notifier::run_mysql_notifications; #[cfg(feature = "postgres")] use crate::client::notifier::run_notifications; #[cfg(feature = "sqlite")] @@ -28,8 +30,8 @@ type ShutdownSignal = std::pin::Pin + S impl Client { /// Starts configured queues and returns a lifecycle handle. /// - /// The client supervises its services: a notification listener, SQLite - /// outbox poller, maintenance, or extension service that fails is logged + /// The client supervises its services: a notification listener, MySQL or + /// SQLite outbox poller, maintenance, or extension service that fails is logged /// and restarted with backoff, and producers keep polling meanwhile, so a /// database outage never stops the client. Only a failure of the /// producers or the completer, which would leave jobs unworked or @@ -399,6 +401,14 @@ impl Supervisor { inner.queue_notifications.clone(), ready, )), + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => Box::pin(run_mysql_notifications( + Arc::clone(&inner), + pool.clone(), + cancel.clone(), + inner.queue_notifications.clone(), + ready, + )), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => Box::pin(run_sqlite_notifications( Arc::clone(&inner), diff --git a/rust/riverqueue/src/client/tests.rs b/rust/riverqueue/src/client/tests.rs index b87f6465c..940a415d9 100644 --- a/rust/riverqueue/src/client/tests.rs +++ b/rust/riverqueue/src/client/tests.rs @@ -151,7 +151,9 @@ async fn go_notification_fixtures_dispatch_correctly() { // in both Postgres-only and SQLite-only builds. #[cfg(feature = "postgres")] let pool = sqlx::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); - #[cfg(not(feature = "postgres"))] + #[cfg(all(not(feature = "postgres"), feature = "mysql"))] + let pool = sqlx::MySqlPool::connect_lazy("mysql://localhost/unused").unwrap(); + #[cfg(all(not(feature = "postgres"), not(feature = "mysql"), feature = "sqlite"))] let pool = sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(); for name in names { diff --git a/rust/riverqueue/src/database.rs b/rust/riverqueue/src/database.rs index e02832147..0a4855cbe 100644 --- a/rust/riverqueue/src/database.rs +++ b/rust/riverqueue/src/database.rs @@ -11,6 +11,8 @@ use std::time::Duration; #[cfg(feature = "postgres")] use jiff::civil::Time; pub use riverqueue_migrate::{SchemaName, SchemaNameError}; +#[cfg(feature = "mysql")] +use sqlx::{MySql, MySqlConnection, MySqlPool}; #[cfg(feature = "postgres")] use sqlx::{PgConnection, PgPool, Postgres}; #[cfg(feature = "sqlite")] @@ -58,6 +60,29 @@ pub async fn begin_postgres(pool: &PgPool) -> Result Result, sqlx::Error> { + let pool = pool.clone(); + run_to_completion(async move { + pool.begin_with("SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN") + .await + }) + .await +} /// Begins a SQLite transaction that may write, protected like /// `begin_postgres` against being abandoned half-started. /// @@ -111,6 +136,8 @@ async fn run_to_completion( } pub(crate) mod erased; +#[cfg(feature = "mysql")] +pub(crate) mod mysql; #[cfg(feature = "postgres")] pub(crate) mod postgres_capabilities; #[cfg(feature = "sqlite")] @@ -130,6 +157,9 @@ pub enum ClientDatabase<'a> { /// A Postgres client's pool and options. #[cfg(feature = "postgres")] Postgres(&'a PostgresDatabase), + /// A MySQL client's pool. + #[cfg(feature = "mysql")] + MySql(&'a MySqlDatabase), /// A SQLite client's pool. #[cfg(feature = "sqlite")] Sqlite(&'a SqliteDatabase), @@ -143,11 +173,25 @@ impl<'a> ClientDatabase<'a> { pub const fn as_postgres(&self) -> Option<&'a PostgresDatabase> { match self { Self::Postgres(database) => Some(database), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, #[cfg(feature = "sqlite")] Self::Sqlite(_) => None, } } + /// Returns the MySQL pool, or `None` for another backend. + #[cfg(feature = "mysql")] + #[must_use] + pub const fn as_mysql(&self) -> Option<&'a MySqlDatabase> { + match self { + #[cfg(feature = "postgres")] + Self::Postgres(_) => None, + Self::MySql(database) => Some(database), + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => None, + } + } /// Returns the SQLite pool, or `None` for another backend. #[cfg(feature = "sqlite")] #[must_use] @@ -156,6 +200,8 @@ impl<'a> ClientDatabase<'a> { #[cfg(feature = "postgres")] Self::Postgres(_) => None, Self::Sqlite(database) => Some(database), + #[cfg(feature = "mysql")] + Self::MySql(_) => None, } } @@ -165,6 +211,8 @@ impl<'a> ClientDatabase<'a> { match self { #[cfg(feature = "postgres")] Self::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + Self::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] Self::Sqlite(_) => DatabaseKind::Sqlite, } @@ -181,6 +229,10 @@ pub enum DatabaseKind { /// SQLite and compatible implementations. #[cfg(feature = "sqlite")] Sqlite, + // Append backends to preserve existing discriminants. + /// MySQL and compatible implementations. + #[cfg(feature = "mysql")] + MySql, } impl fmt::Display for DatabaseKind { @@ -188,6 +240,8 @@ impl fmt::Display for DatabaseKind { formatter.write_str(match self { #[cfg(feature = "postgres")] Self::Postgres => "Postgres", + #[cfg(feature = "mysql")] + Self::MySql => "MySQL", #[cfg(feature = "sqlite")] Self::Sqlite => "SQLite", }) @@ -413,6 +467,12 @@ impl Default for PostgresReindexSchedule { } } +/// A MySQL source and its backend-specific River options. +#[cfg(feature = "mysql")] +#[derive(Clone)] +pub struct MySqlDatabase { + pool: MySqlPool, +} /// A SQLite source and its backend-specific River options. #[cfg(feature = "sqlite")] #[derive(Clone)] @@ -420,6 +480,20 @@ pub struct SqliteDatabase { pool: SqlitePool, } +#[cfg(feature = "mysql")] +impl MySqlDatabase { + /// Uses a MySQL pool. + #[must_use] + pub const fn new(pool: MySqlPool) -> Self { + Self { pool } + } + + /// Returns the underlying SQLx pool. + #[must_use] + pub const fn pool(&self) -> &MySqlPool { + &self.pool + } +} #[cfg(feature = "sqlite")] impl SqliteDatabase { /// Uses a SQLite pool. @@ -435,6 +509,14 @@ impl SqliteDatabase { } } +#[cfg(feature = "mysql")] +impl fmt::Debug for MySqlDatabase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("MySqlDatabase") + .finish_non_exhaustive() + } +} #[cfg(feature = "sqlite")] impl fmt::Debug for SqliteDatabase { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -483,12 +565,20 @@ impl IntoDatabase for &PgPool {} impl IntoDatabase for PostgresDatabase {} #[cfg(feature = "postgres")] impl IntoDatabase for &PostgresDatabase {} +#[cfg(feature = "mysql")] +impl IntoDatabase for MySqlPool {} #[cfg(feature = "sqlite")] impl IntoDatabase for SqlitePool {} +#[cfg(feature = "mysql")] +impl IntoDatabase for &MySqlPool {} #[cfg(feature = "sqlite")] impl IntoDatabase for &SqlitePool {} +#[cfg(feature = "mysql")] +impl IntoDatabase for MySqlDatabase {} #[cfg(feature = "sqlite")] impl IntoDatabase for SqliteDatabase {} +#[cfg(feature = "mysql")] +impl IntoDatabase for &MySqlDatabase {} #[cfg(feature = "sqlite")] impl IntoDatabase for &SqliteDatabase {} @@ -551,6 +641,11 @@ impl<'executor> DatabaseTransactionExecutor<'executor> for &'executor mut sqlx::Transaction<'_, sqlx::Postgres> { } +#[cfg(feature = "mysql")] +impl<'executor> DatabaseTransactionExecutor<'executor> + for &'executor mut sqlx::Transaction<'_, sqlx::MySql> +{ +} #[cfg(feature = "sqlite")] impl<'executor> DatabaseTransactionExecutor<'executor> for &'executor mut sqlx::Transaction<'_, sqlx::Sqlite> @@ -573,6 +668,8 @@ pub(crate) fn into_database(database: D) -> Database { pub(crate) enum DatabasePool<'pool> { #[cfg(feature = "postgres")] Postgres(&'pool PgPool), + #[cfg(feature = "mysql")] + MySql(&'pool MySqlPool), #[cfg(feature = "sqlite")] Sqlite(&'pool SqlitePool), } @@ -582,6 +679,8 @@ mod private { Database, DatabaseConnection, DatabaseInner, DatabaseKind, ErasedExecutor, ErasedTransaction, ExecutorInner, PoolConnection, Transaction, }; + #[cfg(feature = "mysql")] + use super::{MySql, MySqlConnection, MySqlDatabase, MySqlPool}; #[cfg(feature = "postgres")] use super::{PgConnection, PgPool, Postgres, PostgresDatabase}; #[cfg(feature = "sqlite")] @@ -652,6 +751,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for MySqlPool { + fn erase(self) -> Database { + MySqlDatabase::new(self).erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for SqlitePool { fn erase(self) -> Database { @@ -659,6 +764,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for &MySqlPool { + fn erase(self) -> Database { + self.clone().erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for &SqlitePool { fn erase(self) -> Database { @@ -666,6 +777,14 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for MySqlDatabase { + fn erase(self) -> Database { + Database { + inner: DatabaseInner::MySql(self), + } + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for SqliteDatabase { fn erase(self) -> Database { @@ -675,6 +794,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl IntoDatabaseSealed for &MySqlDatabase { + fn erase(self) -> Database { + self.clone().erase() + } + } #[cfg(feature = "sqlite")] impl IntoDatabaseSealed for &SqliteDatabase { fn erase(self) -> Database { @@ -719,6 +844,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut MySqlConnection { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseConnection::MySql(self)) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut SqliteConnection { fn erase(self) -> ErasedExecutor<'executor> { @@ -726,6 +857,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut PoolConnection { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseConnection::MySql(self.as_mut())) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut PoolConnection { fn erase(self) -> ErasedExecutor<'executor> { @@ -733,6 +870,12 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor MySqlPool { + fn erase(self) -> ErasedExecutor<'executor> { + pool_executor(DatabaseKind::MySql) + } + } #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor SqlitePool { fn erase(self) -> ErasedExecutor<'executor> { @@ -740,6 +883,22 @@ mod private { } } + #[cfg(feature = "mysql")] + impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut Transaction<'_, MySql> { + fn erase(self) -> ErasedExecutor<'executor> { + connection_executor(DatabaseTransactionExecutorSealed::connection(self)) + } + } + + #[cfg(feature = "mysql")] + impl<'executor> DatabaseTransactionExecutorSealed<'executor> + for &'executor mut Transaction<'_, MySql> + { + fn connection(self) -> DatabaseConnection<'executor> { + DatabaseConnection::MySql(self.as_mut()) + } + } + #[cfg(feature = "sqlite")] impl<'executor> DatabaseExecutorSealed<'executor> for &'executor mut Transaction<'_, Sqlite> { fn erase(self) -> ErasedExecutor<'executor> { diff --git a/rust/riverqueue/src/database/erased.rs b/rust/riverqueue/src/database/erased.rs index a4e9b93c2..0eda71da5 100644 --- a/rust/riverqueue/src/database/erased.rs +++ b/rust/riverqueue/src/database/erased.rs @@ -4,6 +4,8 @@ use std::fmt; +#[cfg(feature = "mysql")] +use super::MySqlDatabase; #[cfg(feature = "sqlite")] use super::SqliteDatabase; use crate::__private::DatabaseConnection; @@ -40,6 +42,8 @@ impl Database { ) { match &mut self.inner { DatabaseInner::Postgres(source) => source.extend_default_reindex_names(names), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => {} #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => {} } @@ -50,6 +54,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => ClientDatabase::Postgres(source), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(source) => ClientDatabase::MySql(source), #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(source) => ClientDatabase::Sqlite(source), } @@ -61,6 +67,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => DatabaseKind::Sqlite, } @@ -76,6 +84,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => source.capabilities().supports_listen_notify(), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => false, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => false, } @@ -88,6 +98,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => source.capabilities().supports_listen_notify(), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => true, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => true, } @@ -97,7 +109,7 @@ impl Database { /// another backend. #[cfg(feature = "postgres")] #[cfg_attr( - not(feature = "sqlite"), + not(any(feature = "sqlite", feature = "mysql")), expect( clippy::unnecessary_wraps, reason = "another backend may be compiled in" @@ -108,6 +120,8 @@ impl Database { ) -> Option<&super::postgres_capabilities::CapabilitiesCache> { match &self.inner { DatabaseInner::Postgres(source) => Some(source.capabilities()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -120,6 +134,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => Some(source.schema()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -127,7 +143,7 @@ impl Database { #[cfg(feature = "postgres")] #[cfg_attr( - not(feature = "sqlite"), + not(any(feature = "sqlite", feature = "mysql")), expect( clippy::unnecessary_wraps, reason = "another backend may be compiled in" @@ -136,6 +152,8 @@ impl Database { pub(crate) fn postgres_reindex(&self) -> Option<&PostgresReindexConfig> { match &self.inner { DatabaseInner::Postgres(source) => Some(source.reindex()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(_) => None, #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(_) => None, } @@ -177,6 +195,21 @@ impl Database { }) } + /// Borrows a cancellation-safe owned MySQL transaction for a companion crate. + /// + /// # Errors + /// Returns a mismatch when this database uses another backend. + #[cfg(feature = "mysql")] + pub fn mysql_transaction<'executor>( + &self, + transaction: &'executor mut crate::database::mysql::Transaction, + ) -> Result, DatabaseMismatch> { + self.check_kind(DatabaseKind::MySql)?; + Ok(ErasedTransaction { + connection: DatabaseConnection::MySql(transaction), + }) + } + /// Borrows a caller-managed transaction's connection after checking that /// it belongs to this database's backend. pub(crate) fn connection<'executor, E>( @@ -206,6 +239,8 @@ impl Database { match &self.inner { #[cfg(feature = "postgres")] DatabaseInner::Postgres(source) => DatabasePool::Postgres(source.pool()), + #[cfg(feature = "mysql")] + DatabaseInner::MySql(source) => DatabasePool::MySql(source.pool()), #[cfg(feature = "sqlite")] DatabaseInner::Sqlite(source) => DatabasePool::Sqlite(source.pool()), } @@ -226,6 +261,8 @@ impl fmt::Debug for Database { pub(crate) enum DatabaseInner { #[cfg(feature = "postgres")] Postgres(PostgresDatabase), + #[cfg(feature = "mysql")] + MySql(MySqlDatabase), #[cfg(feature = "sqlite")] Sqlite(SqliteDatabase), } diff --git a/rust/riverqueue/src/database/mysql.rs b/rust/riverqueue/src/database/mysql.rs new file mode 100644 index 000000000..896fea839 --- /dev/null +++ b/rust/riverqueue/src/database/mysql.rs @@ -0,0 +1,1566 @@ +//! MySQL operation primitives. +//! +//! MySQL deliberately stays behind River's sealed database boundary. These +//! operations mirror River's storage semantics without exposing SQLx executor +//! types through the public API. Multi-step operations are expressed as small +//! primitives so their caller can compose them in one MySQL transaction. + +#![allow( + clippy::needless_raw_string_hashes, + reason = "consistent SQL delimiters make large dialect-specific statements easier to audit" +)] +#![allow( + clippy::struct_field_names, + reason = "leader_id is the cross-language River protocol field name" +)] + +use std::{ + collections::{HashMap, VecDeque}, + time::Duration, +}; + +use jiff::Timestamp; +use serde_json::{Map, Value, value::RawValue}; +use sqlx::{AssertSqlSafe, Connection as _, FromRow, MySql, MySqlConnection, QueryBuilder, Row}; + +use sqlx::mysql::MySqlRow; + +use crate::{ + AttemptError, JobMetadata, JobRow, JobState, METADATA_KEY_UNIQUE_NONCE, Queue, + client::{DecodedJob, FieldErrors, UndecodableJob, go_time_json, tolerant_row}, + query::{JobListKeyset, JobListSqlPart}, +}; + +mod connection; + +pub use connection::{Connection, Transaction}; + +/// MySQL JSON values are projected as bytes so decoding preserves large integers. +pub(crate) const JOB_COLUMNS: &str = r#" + id, attempt, CONCAT(attempted_at, 'Z') AS attempted_at, CAST(attempted_by AS BINARY) AS attempted_by, + CONCAT(created_at, 'Z') AS created_at, CAST(args AS BINARY) AS encoded_args, CAST(errors AS BINARY) AS errors, + CONCAT(finalized_at, 'Z') AS finalized_at, kind, max_attempts, CAST(metadata AS BINARY) AS metadata, + priority, queue, CONCAT(scheduled_at, 'Z') AS scheduled_at, state, CAST(COALESCE(tags, 'null') AS BINARY) AS tags, + unique_key, unique_states +"#; +const QUEUE_COLUMNS: &str = "CONCAT(created_at, 'Z') AS created_at, CAST(metadata AS CHAR) AS metadata, name, CONCAT(paused_at, 'Z') AS paused_at, CONCAT(updated_at, 'Z') AS updated_at"; +const ERRORS_APPENDED: &str = "JSON_ARRAY_APPEND(CASE WHEN errors IS NULL THEN JSON_ARRAY() WHEN JSON_TYPE(errors) = 'ARRAY' THEN errors ELSE JSON_ARRAY(errors) END, '$', CAST(? AS JSON))"; + +/// A short poll interval keeps local wakeups responsive while queue fetch +/// polling remains the durable recovery path. +pub(crate) const DEFAULT_NOTIFICATION_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Maximum number of outbox rows a notification listener reads at once, +/// matching River Go's MySQL listener. +pub(crate) const NOTIFICATION_BATCH_SIZE: i64 = 256; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum BackendError { + #[error("invalid MySQL River row: {0}")] + InvalidRow(String), + #[error(transparent)] + Sqlx(#[from] sqlx::Error), + #[error("invalid MySQL River JSON")] + Json(#[from] serde_json::Error), +} + +/// Reports MySQL failures the way Postgres's reach callers: as SQLx's own +/// error, with stored values River can't decode as [`sqlx::Error::Decode`]. +impl From for sqlx::Error { + fn from(error: BackendError) -> Self { + match error { + BackendError::Sqlx(error) => error, + BackendError::InvalidRow(message) => { + Self::Decode(format!("invalid MySQL River row: {message}").into()) + } + BackendError::Json(error) => Self::Decode(Box::new(error)), + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct InsertJob<'a> { + pub attempted_at: Option, + pub attempted_by: &'a [String], + pub attempt: i32, + /// When unset, MySQL's `UTC_TIMESTAMP(6)`, like Go's driver. + pub created_at: Option, + pub encoded_args: &'a serde_json::value::RawValue, + pub errors: &'a [AttemptError], + pub finalized_at: Option, + pub id: Option, + pub kind: &'a str, + pub max_attempts: i32, + pub metadata: &'a JobMetadata, + pub priority: i16, + pub queue: &'a str, + /// When unset, MySQL's `UTC_TIMESTAMP(6)`, like Go's driver. + pub scheduled_at: Option, + pub state: JobState, + pub tags: &'a [String], + pub unique_key: Option<&'a [u8]>, + pub unique_nonce: Option<&'a str>, + pub unique_states: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct InsertedJob { + pub job: JobRow, + pub unique_skipped_as_duplicate: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct ClaimJobs<'a> { + pub client_id: &'a str, + /// Kinds to claim, or `None` for every kind. Ignored when claiming + /// selected IDs. + pub kinds: Option<&'a [String]>, + pub limit: i32, + pub max_attempted_by: i32, + pub now: Timestamp, + pub queue: &'a str, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct ListJobs<'a> { + /// Excludes running jobs before applying the limit, as bulk deletion does. + pub exclude_running: bool, + pub ids: &'a [i64], + pub keyset: JobListKeyset, + pub kinds: &'a [&'a str], + pub limit: i32, + pub metadata: Option<&'a Map>, + pub priorities: &'a [i16], + pub queues: &'a [&'a str], + pub states: &'a [JobState], + pub tags_all: &'a [&'a str], + pub tags_any: &'a [&'a str], +} + +#[derive(Clone, Debug)] +pub(crate) struct CompleteJob<'a> { + pub attempt: Option, + pub error: Option<&'a AttemptError>, + pub finalized_at: Option, + pub id: i64, + pub metadata_updates: Option<&'a Map>, + pub now: Timestamp, + pub scheduled_at: Option, + pub state: JobState, +} + +#[derive(Clone, Debug)] +pub(crate) struct RescueJob<'a> { + pub error: &'a AttemptError, + pub finalized_at: Option, + pub id: i64, + pub scheduled_at: Timestamp, + pub state: JobState, + /// The rescue applies only to a job still running from before this time. + pub stuck_horizon: Timestamp, +} + +/// Job cleaner deletion horizons; `None` keeps that state indefinitely. +#[derive(Clone, Debug)] +pub(crate) struct CleanupJobs<'a> { + pub cancelled_before: Option, + pub completed_before: Option, + pub discarded_before: Option, + pub limit: i32, + pub queues_excluded: &'a [&'a str], + pub queues_included: Option<&'a [&'a str]>, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Leader { + pub elected_at: Timestamp, + pub expires_at: Timestamp, + pub leader_id: String, +} + +#[derive(Clone, Debug, FromRow, PartialEq)] +pub(crate) struct Notification { + pub id: i64, + pub payload: String, + pub topic: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct NotificationInput<'a> { + pub payload: &'a str, + pub topic: &'a str, +} + +// River Go stores `attempt`, `max_attempts`, and `priority` as native +// integers, so they decode as `i64`. A value `JobRow`'s field can't hold +// makes the row undecodable, like other columns River can't represent. +// JSON columns are read as bytes, since a value changed out of band might not +// even be UTF-8. +#[derive(Clone, Debug, FromRow)] +struct JobRecord { + attempt: i64, + attempted_at: Option>, + attempted_by: Option>, + created_at: sqlx::types::Text, + encoded_args: Vec, + errors: Option>, + finalized_at: Option>, + id: i64, + kind: String, + max_attempts: i64, + metadata: Vec, + priority: i64, + queue: String, + scheduled_at: sqlx::types::Text, + state: String, + tags: Vec, + unique_key: Option>, + unique_states: Option, +} + +impl JobRecord { + /// Decodes the row, failing if any field can't be decoded. + fn into_job(self) -> Result { + let id = self.id; + self.decode() + .map_err(|job| BackendError::InvalidRow(format!("job {id}: {}", job.error))) + } + + /// Decodes the row, keeping the fields that can be decoded when others + /// can't, like River Go. JSON columns can be changed to any shape, so + /// each decodes on its own. River Go reads the metadata as raw JSON, but a + /// [`JobRow`] can only represent an object. + fn decode(self) -> DecodedJob { + let unidentifiable = |error: String| UndecodableJob { + columns: Vec::new(), + error: format!("job {}: {error}", self.id), + row: None, + }; + let state = JobState::try_from(self.state.as_str()) + .map_err(|error| unidentifiable(error.to_string()))?; + + let mut errors = FieldErrors::default(); + // Like River Go, args that aren't valid JSON are left empty, as + // `null`, so the job isn't worked with them. + let encoded_args = errors.field_or_else( + "args", + json_text(self.encoded_args) + .and_then(|text| RawValue::from_string(text).map_err(|error| error.to_string())), + || RawValue::from_string("null".to_owned()).expect("null is valid JSON"), + ); + let attempted_by = errors.field( + "attempted_by", + self.attempted_by + .map(json_text) + .transpose() + .and_then(|text| { + decode_json_strings(text.as_deref()).map_err(|error| error.to_string()) + }), + ); + let attempt_errors = errors.field( + "errors", + self.errors.map(json_text).transpose().and_then(|text| { + text.as_deref() + .map_or_else(|| Ok(Vec::new()), AttemptError::from_json_array_lenient) + .map_err(|error| error.to_string()) + }), + ); + let metadata = errors.field( + "metadata", + json_text(self.metadata).and_then(|text| { + text.parse::() + .map_err(|error| error.to_string()) + }), + ); + let tags = errors.field( + "tags", + json_text(self.tags).and_then(|text| { + decode_json_strings(Some(&text)).map_err(|error| error.to_string()) + }), + ); + let unique_states = errors.field( + "unique_states", + self.unique_states.map(decode_unique_states).transpose(), + ); + let attempt = errors.field("attempt", i32::try_from(self.attempt)); + let max_attempts = errors.field("max_attempts", i32::try_from(self.max_attempts)); + let priority = errors.field("priority", i16::try_from(self.priority)); + errors.finish(JobRow { + id: self.id, + attempt, + attempted_at: self.attempted_at.map(|time| time.0), + attempted_by, + created_at: self.created_at.0, + encoded_args, + errors: attempt_errors, + finalized_at: self.finalized_at.map(|time| time.0), + kind: self.kind, + max_attempts, + metadata, + priority, + queue: self.queue, + scheduled_at: self.scheduled_at.0, + state, + tags, + unique_key: self.unique_key, + unique_states, + }) + } +} + +/// Decodes a row selected with [`JOB_COLUMNS`] on its own. +pub(crate) fn decode_job_row(row: &MySqlRow) -> DecodedJob { + JobRecord::from_row(row) + .map_err(|error| UndecodableJob { + columns: Vec::new(), + error: error.to_string(), + row: None, + })? + .decode() +} + +#[derive(Clone, Debug, FromRow)] +struct QueueRecord { + created_at: sqlx::types::Text, + metadata: String, + name: String, + paused_at: Option>, + updated_at: sqlx::types::Text, +} + +impl QueueRecord { + fn into_queue(self) -> Result { + let metadata: Value = serde_json::from_str(&self.metadata)?; + let metadata = metadata.as_object().cloned().ok_or_else(|| { + BackendError::InvalidRow(format!("queue {:?} metadata is not an object", self.name)) + })?; + Ok(Queue { + created_at: self.created_at.0, + metadata, + metadata_text: self.metadata, + name: self.name, + paused_at: self.paused_at.map(|time| time.0), + updated_at: self.updated_at.0, + }) + } +} + +#[derive(Clone, Debug, FromRow)] +struct LeaderRecord { + elected_at: sqlx::types::Text, + expires_at: sqlx::types::Text, + leader_id: String, +} + +impl From for Leader { + fn from(record: LeaderRecord) -> Self { + Self { + elected_at: record.elected_at.0, + expires_at: record.expires_at.0, + leader_id: record.leader_id, + } + } +} + +/// Converts a JSON column read as bytes to text. +fn json_text(bytes: Vec) -> Result { + String::from_utf8(bytes).map_err(|error| error.to_string()) +} + +fn decode_json_or_default(encoded: Option<&str>) -> Result +where + T: serde::de::DeserializeOwned + Default, +{ + encoded.map_or_else(|| Ok(T::default()), serde_json::from_str) +} + +/// Decodes a JSON array of strings like Go's `encoding/json` decodes a +/// `[]string`: `null` is empty, as is a `null` element. +fn decode_json_strings(encoded: Option<&str>) -> Result, serde_json::Error> { + Ok( + decode_json_or_default::>>>(encoded)? + .unwrap_or_default() + .into_iter() + .map(Option::unwrap_or_default) + .collect(), + ) +} + +fn decode_unique_states(bits: i64) -> Result, String> { + let bits = u8::try_from(bits).map_err(|_| format!("value out of range for byte: {bits}"))?; + Ok(JobState::ALL + .iter() + .copied() + .filter(|state| bits & state.unique_bit() != 0) + .collect()) +} + +pub(crate) fn mysql_time(time: Timestamp) -> String { + time.strftime("%Y-%m-%d %H:%M:%S%.6f").to_string() +} + +fn mysql_time_optional(time: Option) -> Option { + time.map(mysql_time) +} + +pub(crate) async fn insert( + connection: &mut MySqlConnection, + params: &InsertJob<'_>, +) -> Result { + let mut metadata = params.metadata.clone(); + if let Some(nonce) = params.unique_nonce { + metadata.insert(METADATA_KEY_UNIQUE_NONCE, Value::String(nonce.to_owned()))?; + } + let result = insert_row(connection, params, &metadata, true).await?; + let id = i64::try_from(result.last_insert_id()) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let job = get_locked(connection, id) + .await? + .ok_or(sqlx::Error::RowNotFound)?; + let unique_skipped_as_duplicate = params.unique_nonce.is_some_and(|nonce| { + job.metadata + .get::(METADATA_KEY_UNIQUE_NONCE) + .ok() + .flatten() + .as_deref() + != Some(nonce) + }); + if unique_skipped_as_duplicate + && (params.unique_key.is_none() + || params.unique_key != job.unique_key.as_deref() + || params + .unique_states + .is_none_or(|bits| bits & params.state.unique_bit() == 0) + || !job + .unique_states + .as_ref() + .is_some_and(|states| states.contains(&job.state))) + { + // The no-op conflict update also matches unrelated unique indexes. + // Repeating the plain insert preserves MySQL's constraint name and + // error code for adapters that own those indexes. + insert_row(connection, params, &metadata, false).await?; + return Err(BackendError::InvalidRow( + "conflicting job disappeared while retrying its insert".into(), + )); + } + Ok(InsertedJob { + job, + unique_skipped_as_duplicate, + }) +} + +async fn insert_row( + connection: &mut MySqlConnection, + params: &InsertJob<'_>, + metadata: &JobMetadata, + accept_unique_conflicts: bool, +) -> Result { + let mut sql = String::from( + "INSERT INTO river_job (id, args, attempt, attempted_at, attempted_by, created_at, errors, \ + finalized_at, kind, max_attempts, metadata, priority, queue, scheduled_at, state, tags, unique_key, unique_states) \ + VALUES (?, CAST(? AS JSON), ?, ?, CAST(? AS JSON), COALESCE(?, UTC_TIMESTAMP(6)), CAST(? AS JSON), \ + ?, ?, ?, CAST(? AS JSON), ?, ?, COALESCE(?, UTC_TIMESTAMP(6)), ?, CAST(? AS JSON), ?, ?)", + ); + if accept_unique_conflicts { + sql.push_str(" ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id)"); + } + sqlx::query(AssertSqlSafe(sql)) + .bind(params.id).bind(params.encoded_args.get()).bind(params.attempt) + .bind(mysql_time_optional(params.attempted_at)).bind(serde_json::to_string(params.attempted_by)?) + .bind(mysql_time_optional(params.created_at)).bind(serde_json::to_string(params.errors)?) + .bind(mysql_time_optional(params.finalized_at)).bind(params.kind).bind(params.max_attempts) + .bind(serde_json::to_string(&metadata)?).bind(params.priority).bind(params.queue) + .bind(mysql_time_optional(params.scheduled_at)).bind(params.state.as_str()) + .bind(serde_json::to_string(params.tags)?).bind(params.unique_key).bind(params.unique_states.map(i16::from)) + .execute(connection).await.map_err(Into::into) +} + +pub(crate) async fn claim( + connection: &mut MySqlConnection, + params: &ClaimJobs<'_>, +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + let mut query = QueryBuilder::::new("SELECT id FROM river_job WHERE queue = "); + query.push_bind(params.queue).push(" AND CAST(queue AS BINARY) = ").push_bind(params.queue) + .push(" AND state = 'available' AND scheduled_at <= ").push_bind(mysql_time(params.now)) + .push(" AND NOT EXISTS (SELECT 1 FROM river_queue WHERE name = river_job.queue AND CAST(name AS BINARY) = CAST(river_job.queue AS BINARY) AND paused_at IS NOT NULL)"); + if let Some(kinds) = params.kinds { + if kinds.is_empty() { + return Ok(Vec::new()); + } + query.push(" AND CAST(kind AS BINARY) IN ("); + let mut separated = query.separated(", "); + for kind in kinds { + separated.push_bind(kind); + } + separated.push_unseparated(")"); + } + query + .push(" ORDER BY priority, scheduled_at, id LIMIT ") + .push_bind(params.limit) + .push(" FOR UPDATE SKIP LOCKED"); + let ids = query + .build_query_scalar::() + .fetch_all(&mut *connection) + .await?; + claim_selected(connection, params, &ids).await +} + +/// Updates the selected rows while the caller holds their transaction locks. +pub(crate) async fn claim_selected( + connection: &mut MySqlConnection, + params: &ClaimJobs<'_>, + ids: &[i64], +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + let mut rows = Vec::with_capacity(ids.len()); + for id in ids { + let sql = format!( + "SELECT {JOB_COLUMNS} FROM river_job WHERE id = ? AND state = 'available' FOR UPDATE" + ); + let Some(row) = sqlx::query(AssertSqlSafe(sql)) + .bind(id) + .fetch_optional(&mut *connection) + .await? + else { + continue; + }; + // A malformed attempted_by is retained so the worker fails that row alone. + let attempted_by = row + .try_get::>, _>("attempted_by")? + .as_deref() + .map(serde_json::from_slice::>>>) + .transpose(); + let attempted_by = attempted_by + .ok() + .map(|values| { + let mut values = values.flatten().unwrap_or_default(); + let keep = + usize::try_from(params.max_attempted_by.saturating_sub(1)).unwrap_or_default(); + values.drain(..values.len().saturating_sub(keep)); + values.push(Some(params.client_id.to_owned())); + serde_json::to_string(&values) + }) + .transpose()?; + sqlx::query("UPDATE river_job SET attempt = attempt + 1, attempted_at = ?, attempted_by = COALESCE(CAST(? AS JSON), attempted_by), state = 'running' WHERE id = ?") + .bind(mysql_time(params.now)).bind(attempted_by).bind(id).execute(&mut *connection).await?; + if let Some(row) = get_raw(connection, *id, false).await? { + rows.push(decode_job_row(&row)); + } + } + Ok(rows) +} + +async fn get_raw( + connection: &mut MySqlConnection, + id: i64, + lock: bool, +) -> Result, BackendError> { + let sql = format!( + "SELECT {JOB_COLUMNS} FROM river_job WHERE id = ?{}", + if lock { " FOR UPDATE" } else { "" } + ); + Ok(sqlx::query(AssertSqlSafe(sql)) + .bind(id) + .fetch_optional(&mut *connection) + .await?) +} + +pub(crate) async fn get( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + get_raw(connection, id, false) + .await? + .as_ref() + .map(|row| JobRecord::from_row(row)?.into_job()) + .transpose() +} + +pub(crate) async fn get_locked( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + get_raw(connection, id, true) + .await? + .as_ref() + .map(|row| JobRecord::from_row(row)?.into_job()) + .transpose() +} + +/// Merge objects shallowly, retaining JSON null values as Postgres does. +fn push_metadata( + query: &mut QueryBuilder, + updates: &Map, +) -> Result<(), BackendError> { + if updates.is_empty() { + query.push("metadata"); + return Ok(()); + } + query.push("JSON_SET(metadata"); + for (key, value) in updates { + query + .push(", ") + .push_bind(format!("$.{}", serde_json::to_string(key)?)) + .push(", CAST(") + .push_bind(serde_json::to_string(value)?) + .push(" AS JSON)"); + } + query.push(")"); + Ok(()) +} + +pub(crate) async fn cancel( + connection: &mut MySqlConnection, + id: i64, + now: Timestamp, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.finalized_at.is_some() + || matches!( + row.state, + JobState::Cancelled | JobState::Completed | JobState::Discarded + ) + { + return Ok(None); + } + sqlx::query("UPDATE river_job SET finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE ? END, metadata = JSON_SET(metadata, '$.cancel_attempted_at', ?), state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END WHERE id = ?") + .bind(mysql_time(now)).bind(go_time_json(now)).bind(id).execute(&mut *connection).await?; + get(connection, id).await +} + +pub(crate) async fn delete( + connection: &mut MySqlConnection, + id: i64, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.state == JobState::Running { + return Ok(None); + } + sqlx::query("DELETE FROM river_job WHERE id = ?") + .bind(id) + .execute(connection) + .await?; + Ok(Some(row)) +} + +pub(crate) async fn retry( + connection: &mut MySqlConnection, + id: i64, + now: Timestamp, +) -> Result, BackendError> { + let Some(row) = get_locked(connection, id).await? else { + return Ok(None); + }; + if row.state == JobState::Running + || (row.state == JobState::Available && row.scheduled_at <= now) + { + return Ok(None); + } + sqlx::query("UPDATE river_job SET state = 'available', finalized_at = NULL, metadata = JSON_REMOVE(metadata, '$.cancel_attempted_at'), scheduled_at = ?, max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END WHERE id = ?") + .bind(mysql_time(now)).bind(id).execute(&mut *connection).await?; + get(connection, id).await +} + +pub(crate) async fn complete( + connection: &mut MySqlConnection, + params: &CompleteJob<'_>, +) -> Result, BackendError> { + complete_decoded(connection, params) + .await? + .map(|row| row.map_err(|job| BackendError::InvalidRow(job.error))) + .transpose() +} + +pub(crate) async fn complete_decoded( + connection: &mut MySqlConnection, + params: &CompleteJob<'_>, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, params.id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? != "running" { + return Ok(None); + } + let metadata: Value = serde_json::from_slice(&row.try_get::, _>("metadata")?)?; + let cancel = matches!( + params.state, + JobState::Available | JobState::Retryable | JobState::Scheduled + ) && metadata + .as_object() + .is_some_and(|metadata| metadata.contains_key("cancel_attempted_at")); + let mut query = QueryBuilder::::new("UPDATE river_job SET state = "); + query.push_bind(if cancel { + "cancelled" + } else { + params.state.as_str() + }); + if cancel { + query + .push(", finalized_at = ") + .push_bind(mysql_time(params.now)); + } else { + if let Some(attempt) = params.attempt { + query.push(", attempt = ").push_bind(attempt); + } + if let Some(time) = params.finalized_at { + query.push(", finalized_at = ").push_bind(mysql_time(time)); + } + if let Some(time) = params.scheduled_at { + query.push(", scheduled_at = ").push_bind(mysql_time(time)); + } + } + if let Some(error) = params.error { + // This expression has one placeholder; bind through the query builder. + let (before, after) = ERRORS_APPENDED + .split_once('?') + .expect("one error parameter"); + query + .push(", errors = ") + .push(before) + .push_bind(serde_json::to_string(error)?) + .push(after); + } + if let Some(metadata) = params.metadata_updates { + query.push(", metadata = "); + push_metadata(&mut query, metadata)?; + } + query.push(" WHERE id = ").push_bind(params.id); + query.build().execute(&mut *connection).await?; + Ok(get_raw(connection, params.id, false) + .await? + .as_ref() + .map(decode_job_row)) +} + +pub(crate) async fn merge_metadata_if_not_running( + connection: &mut MySqlConnection, + id: i64, + updates: &Map, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? == "running" { + return Ok(None); + } + let mut query = QueryBuilder::::new("UPDATE river_job SET metadata = "); + push_metadata(&mut query, updates)?; + query.push(" WHERE id = ").push_bind(id); + query.build().execute(&mut *connection).await?; + Ok(get_raw(connection, id, false) + .await? + .as_ref() + .map(decode_job_row)) +} + +pub(crate) async fn update( + connection: &mut MySqlConnection, + id: i64, + updates: &Map, +) -> Result, BackendError> { + if get_raw(connection, id, true).await?.is_none() { + return Ok(None); + } + let mut query = QueryBuilder::::new("UPDATE river_job SET metadata = "); + push_metadata(&mut query, updates)?; + query.push(" WHERE id = ").push_bind(id); + query.build().execute(&mut *connection).await?; + get(connection, id).await +} + +#[allow( + clippy::too_many_lines, + reason = "the bound-only query builder keeps all list filters and keyset ordering auditable" +)] +pub(crate) async fn list( + connection: &mut MySqlConnection, + params: &ListJobs<'_>, +) -> Result, BackendError> { + if params.limit <= 0 { + return Ok(Vec::new()); + } + + let mut query = + QueryBuilder::::new(format!("SELECT {JOB_COLUMNS} FROM river_job WHERE true")); + if let Some(after) = params.keyset.after_sql() { + query.push(" AND "); + for part in after { + match part { + JobListSqlPart::AfterId => { + query.push_bind(params.keyset.after_id().expect("cursor has an ID")); + } + JobListSqlPart::AfterTime => { + query.push_bind(mysql_time( + params.keyset.after_time().expect("cursor has a time"), + )); + } + JobListSqlPart::Sql(sql) => { + query.push(sql); + } + } + } + } + if params.exclude_running { + query.push(" AND state != 'running'"); + } + if !params.ids.is_empty() { + query.push(" AND id IN ("); + let mut separated = query.separated(", "); + for id in params.ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + } + if !params.kinds.is_empty() { + query.push(" AND CAST(kind AS BINARY) IN ("); + let mut separated = query.separated(", "); + for kind in params.kinds { + separated.push_bind(kind); + } + separated.push_unseparated(")"); + } + if !params.priorities.is_empty() { + query.push(" AND priority IN ("); + let mut separated = query.separated(", "); + for priority in params.priorities { + separated.push_bind(priority); + } + separated.push_unseparated(")"); + } + if !params.queues.is_empty() { + query.push(" AND CAST(queue AS BINARY) IN ("); + let mut separated = query.separated(", "); + for queue in params.queues { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + if !params.states.is_empty() { + query.push(" AND state IN ("); + let mut separated = query.separated(", "); + for state in params.states { + separated.push_bind(state.as_str()); + } + separated.push_unseparated(")"); + } + if let Some(metadata) = params.metadata { + query.push(" AND "); + push_contains_value( + &mut query, + "metadata", + &Value::Object(metadata.clone()), + &mut 0, + )?; + } + for tag in params.tags_all { + query + .push(" AND JSON_CONTAINS(tags, JSON_QUOTE(") + .push_bind(tag) + .push("))"); + } + if !params.tags_any.is_empty() { + query.push(" AND EXISTS (SELECT 1 FROM JSON_TABLE(tags, '$[*]' COLUMNS(value VARCHAR(255) PATH '$')) AS tag WHERE CAST(value AS BINARY) IN ("); + let mut separated = query.separated(", "); + for tag in params.tags_any { + separated.push_bind(tag); + } + separated.push_unseparated("))"); + } + query + .push(" ORDER BY ") + .push(mysql_order(¶ms.keyset.order_sql())); + query.push(" LIMIT ").push_bind(params.limit); + + let records = query + .build_query_as::() + .fetch_all(&mut *connection) + .await?; + records.into_iter().map(JobRecord::into_job).collect() +} + +fn mysql_order(order: &str) -> String { + let field = order.split_whitespace().next().expect("order has a field"); + if order.contains("NULLS LAST") { + format!("{field} IS NULL ASC, {}", order.replace(" NULLS LAST", "")) + } else if order.contains("NULLS FIRST") { + format!( + "{field} IS NULL DESC, {}", + order.replace(" NULLS FIRST", "") + ) + } else { + order.to_owned() + } +} + +// JSON_CONTAINS flattens nested arrays; Postgres containment preserves each +// level. JSON_TABLE traverses each level explicitly. Scalar subqueries avoid +// MySQL 8 semijoin rewrites losing nested correlations. +fn push_contains_value( + query: &mut QueryBuilder, + source: &str, + fragment: &Value, + aliases: &mut usize, +) -> Result<(), BackendError> { + match fragment { + Value::Object(object) => { + query.push(format!("(JSON_TYPE({source}) = 'OBJECT'")); + for (key, value) in object { + let alias = format!("contains_{aliases}"); + *aliases += 1; + query + .push(format!( + " AND (SELECT 1 FROM JSON_TABLE(JSON_EXTRACT({source}, " + )) + .push_bind(format!("$.{}", serde_json::to_string(key)?)) + .push(format!( + "), '$' COLUMNS(value JSON PATH '$')) AS {alias} WHERE " + )); + push_contains_value(query, &format!("{alias}.value"), value, aliases)?; + query.push(" LIMIT 1) IS NOT NULL"); + } + query.push(")"); + } + Value::Array(array) => { + query.push(format!("(JSON_TYPE({source}) = 'ARRAY'")); + for value in array { + let alias = format!("contains_{aliases}"); + *aliases += 1; + query.push(format!(" AND (SELECT 1 FROM JSON_TABLE({source}, '$[*]' COLUMNS(value JSON PATH '$')) AS {alias} WHERE ")); + push_contains_value(query, &format!("{alias}.value"), value, aliases)?; + query.push(" LIMIT 1) IS NOT NULL"); + } + query.push(")"); + } + Value::Null => { + query.push(format!( + "({source} IS NULL OR JSON_TYPE({source}) = 'NULL')" + )); + } + value => { + query.push(format!("(JSON_TYPE({source}) NOT IN ('ARRAY', 'OBJECT') AND JSON_CONTAINS({source}, CAST(")) + .push_bind(serde_json::to_string(value)?).push(" AS JSON)))"); + } + } + Ok(()) +} + +pub(crate) async fn queue_upsert( + connection: &mut MySqlConnection, + name: &str, + metadata: &Map, + paused_at: Option, + now: Timestamp, +) -> Result { + sqlx::query("INSERT INTO river_queue (created_at, metadata, name, paused_at, updated_at) VALUES (?, CAST(? AS JSON), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?") + .bind(mysql_time(now)).bind(serde_json::to_string(metadata)?).bind(name) + .bind(mysql_time_optional(paused_at)).bind(mysql_time(now)).bind(mysql_time(now)) + .execute(&mut *connection).await?; + queue_get(connection, name) + .await? + .ok_or_else(|| sqlx::Error::RowNotFound.into()) +} + +pub(crate) async fn queue_get( + connection: &mut MySqlConnection, + name: &str, +) -> Result, BackendError> { + sqlx::query_as::<_, QueueRecord>(AssertSqlSafe(format!( + "SELECT {QUEUE_COLUMNS} FROM river_queue WHERE name = ? AND CAST(name AS BINARY) = ?" + ))) + .bind(name) + .bind(name) + .fetch_optional(connection) + .await? + .map(QueueRecord::into_queue) + .transpose() +} + +pub(crate) async fn queue_list( + connection: &mut MySqlConnection, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + sqlx::query_as::<_, QueueRecord>(AssertSqlSafe(format!( + "SELECT {QUEUE_COLUMNS} FROM river_queue ORDER BY name LIMIT ?" + ))) + .bind(limit) + .fetch_all(connection) + .await? + .into_iter() + .map(QueueRecord::into_queue) + .collect() +} + +pub(crate) async fn queue_pause( + connection: &mut MySqlConnection, + name: &str, + now: Timestamp, +) -> Result, BackendError> { + queue_set_paused(connection, name, Some(now), now).await +} + +pub(crate) async fn queue_resume( + connection: &mut MySqlConnection, + name: &str, + now: Timestamp, +) -> Result, BackendError> { + queue_set_paused(connection, name, None, now).await +} + +async fn queue_set_paused( + connection: &mut MySqlConnection, + name: &str, + paused_at: Option, + now: Timestamp, +) -> Result, BackendError> { + let names = sqlx::query_scalar::<_, String>("SELECT name FROM river_queue WHERE (? = '*' OR CAST(name AS BINARY) = ?) ORDER BY name FOR UPDATE") + .bind(name).bind(name).fetch_all(&mut *connection).await?; + let mut queues = Vec::with_capacity(names.len()); + for name in names { + sqlx::query("UPDATE river_queue SET updated_at = CASE WHEN (? AND paused_at IS NULL) OR (NOT ? AND paused_at IS NOT NULL) THEN ? ELSE updated_at END, paused_at = CASE WHEN ? THEN COALESCE(paused_at, ?) ELSE NULL END WHERE name = ?") + .bind(paused_at.is_some()).bind(paused_at.is_some()).bind(mysql_time(now)) + .bind(paused_at.is_some()).bind(mysql_time_optional(paused_at)).bind(&name).execute(&mut *connection).await?; + if let Some(queue) = queue_get(connection, &name).await? { + queues.push(queue); + } + } + Ok(queues) +} + +pub(crate) async fn queue_update( + connection: &mut MySqlConnection, + name: &str, + metadata: Option<&Map>, + now: Timestamp, +) -> Result, BackendError> { + sqlx::query("UPDATE river_queue SET metadata = COALESCE(CAST(? AS JSON), metadata), updated_at = ? WHERE name = ? AND CAST(name AS BINARY) = ?") + .bind(metadata.map(serde_json::to_string).transpose()?).bind(mysql_time(now)).bind(name).bind(name).execute(&mut *connection).await?; + queue_get(connection, name).await +} + +pub(crate) async fn queue_delete_expired( + connection: &mut MySqlConnection, + updated_before: Timestamp, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + let names = sqlx::query_scalar::<_, String>("SELECT name FROM river_queue WHERE updated_at < ? ORDER BY name LIMIT ? FOR UPDATE SKIP LOCKED") + .bind(mysql_time(updated_before)).bind(limit).fetch_all(&mut *connection).await?; + for name in &names { + sqlx::query("DELETE FROM river_queue WHERE name = ?") + .bind(name) + .execute(&mut *connection) + .await?; + } + Ok(names) +} + +pub(crate) async fn leader_elect( + connection: &mut MySqlConnection, + leader_id: &str, + now: Timestamp, + ttl: Duration, +) -> Result, BackendError> { + let expires_at = now + + jiff::SignedDuration::try_from(ttl) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let result = sqlx::query("INSERT INTO river_leader (name, leader_id, elected_at, expires_at) VALUES ('default', ?, ?, ?)") + .bind(leader_id).bind(mysql_time(now)).bind(mysql_time(expires_at)).execute(&mut *connection).await; + match result { + Ok(_) => leader_get(connection).await, + Err(error) + if error + .as_database_error() + .is_some_and(sqlx::error::DatabaseError::is_unique_violation) => + { + Ok(None) + } + Err(error) => Err(error.into()), + } +} + +async fn leader_get(connection: &mut MySqlConnection) -> Result, BackendError> { + Ok(sqlx::query_as::<_, LeaderRecord>("SELECT CONCAT(elected_at, 'Z') AS elected_at, CONCAT(expires_at, 'Z') AS expires_at, leader_id FROM river_leader WHERE name = 'default' FOR UPDATE") + .fetch_optional(connection).await?.map(Leader::from)) +} + +pub(crate) async fn leader_reelect( + connection: &mut MySqlConnection, + leader_id: &str, + elected_at: Timestamp, + now: Timestamp, + ttl: Duration, +) -> Result, BackendError> { + let expires_at = now + + jiff::SignedDuration::try_from(ttl) + .map_err(|error| BackendError::InvalidRow(error.to_string()))?; + let result = sqlx::query("UPDATE river_leader SET expires_at = ? WHERE name = 'default' AND CAST(leader_id AS BINARY) = ? AND elected_at = ? AND expires_at >= ?") + .bind(mysql_time(expires_at)).bind(leader_id).bind(mysql_time(elected_at)).bind(mysql_time(now)).execute(&mut *connection).await?; + if result.rows_affected() == 0 { + return Ok(None); + } + leader_get(connection).await +} + +pub(crate) async fn leader_resign( + connection: &mut MySqlConnection, + leader_id: &str, + elected_at: Timestamp, +) -> Result { + Ok(sqlx::query("DELETE FROM river_leader WHERE name = 'default' AND CAST(leader_id AS BINARY) = ? AND elected_at = ?") + .bind(leader_id).bind(mysql_time(elected_at)).execute(connection).await?.rows_affected() > 0) +} + +pub(crate) async fn leader_delete_expired( + connection: &mut MySqlConnection, + now: Timestamp, +) -> Result { + Ok(sqlx::query("DELETE FROM river_leader WHERE expires_at < ?") + .bind(mysql_time(now)) + .execute(connection) + .await? + .rows_affected()) +} + +pub(crate) async fn notification_insert( + connection: &mut MySqlConnection, + notifications: &[NotificationInput<'_>], +) -> Result { + if notifications.is_empty() { + return Ok(0); + } + let mut query = QueryBuilder::::new("INSERT INTO river_notification (payload, topic) "); + query.push_values(notifications, |mut row, notification| { + row.push_bind(notification.payload) + .push_bind(notification.topic); + }); + Ok(query.build().execute(connection).await?.rows_affected()) +} + +// Ordered locking reads wait for earlier uncommitted IDs. MAX(id), snapshot +// reads and SKIP LOCKED can advance the cursor past a later commit forever. +pub(crate) async fn notification_get_after( + connection: &mut MySqlConnection, + after_id: i64, + topics: &[&str], + limit: i64, +) -> Result, BackendError> { + if limit <= 0 || topics.is_empty() { + return Ok(Vec::new()); + } + Ok(sqlx::query_as::<_, Notification>("SELECT id, payload, topic FROM river_notification FORCE INDEX (PRIMARY) WHERE id > ? ORDER BY id LIMIT ? FOR SHARE") + .bind(after_id).bind(limit).fetch_all(connection).await?) +} + +pub(crate) async fn notification_last_id( + connection: &mut MySqlConnection, +) -> Result { + // Read every ID to lock lower outstanding inserts before establishing the cursor. + Ok(sqlx::query_scalar::<_, i64>( + "SELECT id FROM river_notification FORCE INDEX (PRIMARY) ORDER BY id FOR UPDATE", + ) + .fetch_all(connection) + .await? + .last() + .copied() + .unwrap_or_default()) +} + +#[derive(Debug, Default)] +pub(crate) struct NotificationListener { + /// ID of the last row read, or `None` before connecting. + last_id: Option, + /// Rows read but not yet delivered. + pending: VecDeque, + /// Subscribed topics and the ID after which each one's rows are + /// delivered. + topics: HashMap, +} + +impl NotificationListener { + /// Starts reading after the outbox's current maximum ID. + pub(crate) async fn connect( + &mut self, + connection: &mut MySqlConnection, + ) -> Result<(), BackendError> { + let mut tx = connection.begin().await?; + let last_id = notification_last_id(&mut tx).await?; + tx.commit().await?; + self.last_id = Some(last_id); + Ok(()) + } + + /// Whether the initial outbox cursor has been established. + pub(crate) const fn is_connected(&self) -> bool { + self.last_id.is_some() + } + + /// Subscribes to `topic`, delivering only rows written from now on. + /// Subscribing to a topic that's already subscribed changes nothing. + pub(crate) async fn listen( + &mut self, + connection: &mut MySqlConnection, + topic: &str, + ) -> Result<(), BackendError> { + if self.topics.contains_key(topic) { + return Ok(()); + } + // Only this topic's starting ID advances. Advancing the shared cursor + // could skip undelivered rows of topics that are already subscribed. + let mut tx = connection.begin().await?; + let last_id = notification_last_id(&mut tx).await?; + tx.commit().await?; + // Cleanup may have deleted rows that are still buffered locally. + let start_id = last_id.max(self.last_id.unwrap_or_default()); + self.topics.insert(topic.to_owned(), start_id); + Ok(()) + } + + /// Returns the next notification on a subscribed topic, or `None` once + /// the outbox has no newer rows on subscribed topics. + pub(crate) async fn next( + &mut self, + connection: &mut MySqlConnection, + ) -> Result, BackendError> { + loop { + if self.pending.is_empty() { + let Some(after_id) = self.last_id else { + return Ok(None); + }; + let topics = self.topics.keys().map(String::as_str).collect::>(); + // Lock each bounded scan in its own transaction, then release + // its locks before dispatching notifications. Cursor and buffer + // changes become visible only after a successful commit. + let mut tx = connection.begin().await?; + let notifications = + notification_get_after(&mut tx, after_id, &topics, NOTIFICATION_BATCH_SIZE) + .await?; + tx.commit().await?; + let Some(last) = notifications.last() else { + return Ok(None); + }; + self.last_id = Some(last.id); + self.pending.extend(notifications); + } + // Subscriptions can change between reading a batch and delivering + // it. + while let Some(notification) = self.pending.pop_front() { + if self + .topics + .get(¬ification.topic) + .is_some_and(|start_id| notification.id > *start_id) + { + return Ok(Some(notification)); + } + } + } + } +} + +pub(crate) async fn notification_cleanup( + connection: &mut MySqlConnection, + created_before: Timestamp, + limit: i64, +) -> Result { + if limit <= 0 { + return Ok(0); + } + Ok(sqlx::query( + "DELETE FROM river_notification WHERE created_at < ? ORDER BY created_at, id LIMIT ?", + ) + .bind(mysql_time(created_before)) + .bind(limit) + .execute(&mut *connection) + .await? + .rows_affected()) +} + +pub(crate) async fn stuck_jobs( + connection: &mut MySqlConnection, + after_id: i64, + attempted_before: Timestamp, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + let sql = format!( + r#" + SELECT {JOB_COLUMNS} + FROM river_job + WHERE state = 'running' + AND id > ? + AND attempted_at < ? + ORDER BY id ASC + LIMIT ? + "# + ); + let records = sqlx::query(AssertSqlSafe(sql)) + .bind(after_id) + .bind(mysql_time(attempted_before)) + .bind(limit) + .fetch_all(&mut *connection) + .await?; + Ok(tolerant_rows(&records)) +} + +/// Loads exactly the rescue candidates selected by an exact-version +/// extension. The enclosing transaction protects the subsequent state +/// transition. +pub(crate) async fn jobs_by_ids( + connection: &mut MySqlConnection, + ids: &[i64], +) -> Result, BackendError> { + if ids.is_empty() { + return Ok(Vec::new()); + } + let mut query = + QueryBuilder::::new(format!("SELECT {JOB_COLUMNS} FROM river_job WHERE id IN (")); + { + let mut separated = query.separated(", "); + for id in ids { + separated.push_bind(id); + } + } + query.push(") ORDER BY id ASC"); + let records = query.build().fetch_all(&mut *connection).await?; + Ok(tolerant_rows(&records)) +} + +/// Decodes rows for the rescuer, which like River Go's `JobGetStuck` +/// tolerates undecodable fields so a job stranded by one can be rescued. +fn tolerant_rows(records: &[MySqlRow]) -> Vec { + records + .iter() + .filter_map(|record| tolerant_row(decode_job_row(record))) + .collect() +} + +pub(crate) async fn cleanup_jobs( + connection: &mut MySqlConnection, + params: &CleanupJobs<'_>, +) -> Result { + let horizons = [ + ("cancelled", params.cancelled_before), + ("completed", params.completed_before), + ("discarded", params.discarded_before), + ]; + if params.limit <= 0 || horizons.iter().all(|(_, before)| before.is_none()) { + return Ok(0); + } + let mut query = QueryBuilder::::new("SELECT id FROM river_job WHERE ("); + let mut first = true; + for (state, before) in horizons { + let Some(before) = before else { + continue; + }; + if !first { + query.push(" OR "); + } + first = false; + query + .push("(state = ") + .push_bind(state) + .push(" AND finalized_at < ") + .push_bind(mysql_time(before)) + .push(")"); + } + query.push(")"); + if !params.queues_excluded.is_empty() { + query.push(" AND CAST(queue AS BINARY) NOT IN ("); + let mut separated = query.separated(", "); + for queue in params.queues_excluded { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + if let Some(queues_included) = params.queues_included { + if queues_included.is_empty() { + return Ok(0); + } + query.push(" AND CAST(queue AS BINARY) IN ("); + let mut separated = query.separated(", "); + for queue in queues_included { + separated.push_bind(queue); + } + separated.push_unseparated(")"); + } + query + .push(" ORDER BY id ASC LIMIT ") + .push_bind(params.limit) + .push(" FOR UPDATE SKIP LOCKED"); + let mut tx = connection.begin().await?; + let ids: Vec = query.build_query_scalar().fetch_all(&mut *tx).await?; + let deleted = if ids.is_empty() { + 0 + } else { + let mut delete = QueryBuilder::::new( + "DELETE /*+ JOIN_INDEX(river_job PRIMARY) */ FROM river_job WHERE id IN (", + ); + let mut separated = delete.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + delete.build().execute(&mut *tx).await?.rows_affected() + }; + tx.commit().await?; + Ok(deleted) +} + +/// A due job the scheduler may make available. +#[derive(Clone, Debug, FromRow)] +pub(crate) struct ScheduleCandidate { + pub id: i64, + pub unique_key: Option>, +} + +/// Selects due retryable/scheduled jobs in scheduler order. Scheduling is a +/// multi-step MySQL operation: callers keep a write transaction open while +/// checking unique collisions and applying the transitions below. +/// +/// Like River Go, only the columns scheduling needs are selected, so a job +/// whose row can't be decoded, like one whose attempt failed because a JSON +/// column isn't valid JSON, doesn't fail scheduling for every other job. The +/// transitions below return such a job's row with the undecodable fields left +/// empty. +pub(crate) async fn schedule_candidates( + connection: &mut MySqlConnection, + now: Timestamp, + limit: i32, +) -> Result, BackendError> { + if limit <= 0 { + return Ok(Vec::new()); + } + Ok(sqlx::query_as::<_, ScheduleCandidate>( + r#" + SELECT id, unique_key + FROM river_job + WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= ? + ORDER BY priority ASC, scheduled_at ASC, id ASC + LIMIT ? FOR UPDATE SKIP LOCKED + "#, + ) + .bind(mysql_time(now)) + .bind(limit) + .fetch_all(&mut *connection) + .await?) +} + +pub(crate) async fn schedule_has_unique_collision( + connection: &mut MySqlConnection, + id: i64, + unique_key: &[u8], +) -> Result { + let exists = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM river_job + WHERE id != ? + AND unique_key = ? + AND unique_states IS NOT NULL + AND CASE state + WHEN 'available' THEN unique_states & (1 << 0) + WHEN 'cancelled' THEN unique_states & (1 << 1) + WHEN 'completed' THEN unique_states & (1 << 2) + WHEN 'discarded' THEN unique_states & (1 << 3) + WHEN 'pending' THEN unique_states & (1 << 4) + WHEN 'retryable' THEN unique_states & (1 << 5) + WHEN 'running' THEN unique_states & (1 << 6) + WHEN 'scheduled' THEN unique_states & (1 << 7) + ELSE 0 + END >= 1 + ) + "#, + ) + .bind(id) + .bind(unique_key) + .fetch_one(&mut *connection) + .await?; + Ok(exists) +} + +pub(crate) async fn rescue( + connection: &mut MySqlConnection, + params: &RescueJob<'_>, +) -> Result, BackendError> { + let Some(row) = get_raw(connection, params.id, true).await? else { + return Ok(None); + }; + if row.try_get::("state")? != "running" + || row + .try_get::>, _>("attempted_at")? + .map(|time| time.0) + .is_none_or(|time| time >= params.stuck_horizon) + { + return Ok(None); + } + let sql = format!( + "UPDATE river_job SET errors = {ERRORS_APPENDED}, finalized_at = ?, scheduled_at = ?, \ + metadata = JSON_SET(metadata, '$.\"river:rescue_count\"', CASE WHEN JSON_TYPE(JSON_EXTRACT(metadata, '$.\"river:rescue_count\"')) IN ('INTEGER', 'DOUBLE', 'DECIMAL') THEN JSON_EXTRACT(metadata, '$.\"river:rescue_count\"') + 1 ELSE 1 END), state = ? WHERE id = ?" + ); + sqlx::query(AssertSqlSafe(sql)) + .bind(serde_json::to_string(params.error)?) + .bind(mysql_time_optional(params.finalized_at)) + .bind(mysql_time(params.scheduled_at)) + .bind(params.state.as_str()) + .bind(params.id) + .execute(&mut *connection) + .await?; + Ok(get_raw(connection, params.id, false) + .await? + .as_ref() + .and_then(|row| tolerant_row(decode_job_row(row)))) +} + +pub(crate) async fn schedule_set_available( + connection: &mut MySqlConnection, + ids: &[i64], +) -> Result, BackendError> { + let mut available = Vec::with_capacity(ids.len()); + for id in ids { + match sqlx::query("UPDATE river_job SET state = 'available' WHERE id = ? AND state IN ('scheduled', 'retryable')").bind(id).execute(&mut *connection).await { + Ok(result) if result.rows_affected() > 0 => { + if let Some(row) = get_raw(connection, *id, false).await?.as_ref().and_then(|row| tolerant_row(decode_job_row(row))) { available.push(row); } + } + Ok(_) => {} + Err(error) if error.as_database_error().is_some_and(sqlx::error::DatabaseError::is_unique_violation) => { + // A concurrent unique insert may win after the scheduler's + // collision check. InnoDB rolls back this statement only. + schedule_discard_conflicts(connection, &[*id], Timestamp::now()).await?; + } + Err(error) => return Err(error.into()), + } + } + Ok(available) +} + +pub(crate) async fn schedule_discard_conflicts( + connection: &mut MySqlConnection, + ids: &[i64], + now: Timestamp, +) -> Result, BackendError> { + for id in ids { + sqlx::query("UPDATE river_job SET metadata = JSON_SET(metadata, '$.unique_key_conflict', 'scheduler_discarded'), finalized_at = ?, state = 'discarded' WHERE id = ?") + .bind(mysql_time(now)).bind(id).execute(&mut *connection).await?; + } + jobs_by_ids(connection, ids).await +} + +#[cfg(all(test, river_mysql_tests))] +mod tests; diff --git a/rust/riverqueue/src/database/mysql/connection.rs b/rust/riverqueue/src/database/mysql/connection.rs new file mode 100644 index 000000000..a1de01920 --- /dev/null +++ b/rust/riverqueue/src/database/mysql/connection.rs @@ -0,0 +1,114 @@ +//! Pooled MySQL connections that are discarded when an operation is cancelled. + +use std::ops::{Deref, DerefMut}; + +use sqlx::{ + MySql, MySqlConnection, MySqlPool, mysql::MySqlTransactionManager, pool::PoolConnection, +}; +use sqlx_core::transaction::TransactionManager; + +/// Returns a connection to the pool only after its operation finishes. +/// +/// SQLx's MySQL protocol recovery can hang if a future is dropped while a +/// statement is being prepared. Returning that connection to the pool then +/// leaves its release-time ping waiting forever, including during pool close. +/// Closing the socket on cancellation also rolls back any open transaction. +#[derive(Debug)] +pub struct Connection(Option>); + +impl Connection { + /// Acquires a connection that is discarded unless explicitly released. + /// + /// # Errors + /// Returns the pool's connection acquisition error. + pub async fn acquire(pool: &MySqlPool) -> Result { + Ok(Self(Some(pool.acquire().await?))) + } + + /// Returns a connection whose operation completed to the caller's pool. + pub fn release(mut self) { + drop(self.0.take()); + } +} + +impl Deref for Connection { + type Target = MySqlConnection; + + fn deref(&self) -> &Self::Target { + self.0.as_deref().expect("connection has not been released") + } +} + +impl DerefMut for Connection { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + .as_deref_mut() + .expect("connection has not been released") + } +} + +impl Drop for Connection { + fn drop(&mut self) { + // A graceful close or the pool's release-time ping could wait on the + // abandoned statement. Detaching and dropping closes the socket instead. + drop(self.0.take().map(PoolConnection::detach)); + } +} + +/// A River-owned transaction that discards its connection unless committed. +#[derive(Debug)] +pub struct Transaction(Connection); + +impl Transaction { + /// Begins a transaction whose connection is discarded on cancellation. + /// + /// # Errors + /// Returns an acquisition or transaction start error. + pub async fn begin(pool: &MySqlPool) -> Result { + let mut connection = Connection::acquire(pool).await?; + // Use SQLx's transaction manager so extension savepoints see the + // correct transaction depth, just as with a SQLx Transaction. + MySqlTransactionManager::begin( + &mut connection, + Some(sqlx::SqlStr::from_static( + "SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN", + )), + ) + .await?; + Ok(Self(connection)) + } + + /// Rolls back a completed operation and returns its connection to the pool. + /// + /// # Errors + /// Returns a database rollback error. + pub async fn rollback(mut self) -> Result<(), sqlx::Error> { + MySqlTransactionManager::rollback(&mut self.0).await?; + self.0.release(); + Ok(()) + } + + /// Commits a completed operation and returns its connection to the pool. + /// + /// # Errors + /// Returns a database commit error. + pub async fn commit(mut self) -> Result<(), sqlx::Error> { + MySqlTransactionManager::commit(&mut self.0).await?; + self.0.release(); + Ok(()) + } +} + +impl Deref for Transaction { + type Target = MySqlConnection; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Transaction { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/rust/riverqueue/src/database/mysql/tests.rs b/rust/riverqueue/src/database/mysql/tests.rs new file mode 100644 index 000000000..7a88da865 --- /dev/null +++ b/rust/riverqueue/src/database/mysql/tests.rs @@ -0,0 +1,928 @@ +use std::{ + collections::HashSet, + str::FromStr, + sync::atomic::{AtomicU64, Ordering}, +}; + +use riverqueue_migrate::{Direction, MigrateOpts, MySqlMigrator}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sqlx::{ + MySqlPool, + mysql::{MySqlConnectOptions, MySqlPoolOptions}, +}; + +use super::*; +use crate::{Client, InsertOpts, JobArgs, JobListParams, UniqueOpts}; + +mod conformance; + +#[derive(Debug, Deserialize, JobArgs, Serialize)] +#[river(kind = "mysql_test")] +struct Args { + value: i64, +} + +struct TestDatabase { + admin: MySqlPool, + client: Client, + name: String, + pool: MySqlPool, +} + +impl TestDatabase { + async fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let url = std::env::var("RIVER_RUST_MYSQL_URL") + .expect("RIVER_RUST_MYSQL_URL is required for MySQL tests"); + let options = MySqlConnectOptions::from_str(&url).unwrap(); + let admin = MySqlPoolOptions::new() + .max_connections(1) + .connect_with(options.clone()) + .await + .unwrap(); + let name = format!( + "river_rust_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + sqlx::query(AssertSqlSafe(format!("CREATE DATABASE `{name}`"))) + .execute(&admin) + .await + .unwrap(); + let pool = MySqlPoolOptions::new() + .max_connections(12) + .connect_with(options.database(&name)) + .await + .unwrap(); + MySqlMigrator::new(pool.clone()).migrate_up().await.unwrap(); + let client = Client::builder(pool.clone()).build().unwrap(); + Self { + admin, + client, + name, + pool, + } + } + + async fn close(self) { + tokio::time::timeout(Duration::from_secs(10), self.pool.close()) + .await + .expect("MySQL test pool did not close"); + sqlx::query(AssertSqlSafe(format!("DROP DATABASE `{}`", self.name))) + .execute(&self.admin) + .await + .unwrap(); + self.admin.close().await; + } +} + +#[tokio::test] +async fn cancelled_operations_do_not_block_pool_close() { + use crate::storage::{Access, Session}; + + for access in [Access::Autocommit, Access::Transaction] { + let db = TestDatabase::new().await; + // Keep the blocker outside the pool so closing the pool only waits + // for the operation under test, while its statement is still blocked. + let mut blocker = db.pool.acquire().await.unwrap().detach(); + sqlx::raw_sql("LOCK TABLES river_job WRITE") + .execute(&mut blocker) + .await + .unwrap(); + + let mut session = Session::begin(&db.client.inner.database, access) + .await + .unwrap(); + let client = db.client.clone(); + let query = tokio::spawn(async move { + session.storage(&client.inner).job_get(1).await.unwrap(); + session.commit().await.unwrap(); + }); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM performance_schema.threads \ + WHERE PROCESSLIST_DB = ? AND PROCESSLIST_STATE = 'Waiting for table metadata lock'", + ) + .bind(&db.name) + .fetch_one(&db.admin) + .await + .unwrap(); + if waiting > 0 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("query did not block while preparing its statement"); + + query.abort(); + assert!(query.await.unwrap_err().is_cancelled()); + let closed = tokio::time::timeout(Duration::from_secs(2), db.pool.close()).await; + sqlx::raw_sql("UNLOCK TABLES") + .execute(&mut blocker) + .await + .unwrap(); + drop(blocker); + db.close().await; + closed.expect("pool close waited for a cancelled query's table lock"); + } +} + +#[tokio::test] +async fn cleanup_skips_locked_jobs() { + let db = TestDatabase::new().await; + let running = db + .client + .insert(Args { value: 1 }) + .await + .unwrap() + .job + .row + .id; + let locked = db + .client + .insert(Args { value: 2 }) + .await + .unwrap() + .job + .row + .id; + let expired = db + .client + .insert(Args { value: 3 }) + .await + .unwrap() + .job + .row + .id; + for value in 4..1003 { + db.client.insert(Args { value }).await.unwrap(); + } + sqlx::query("UPDATE river_job SET state='completed', finalized_at='1970-01-01' WHERE id >= ?") + .bind(locked) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("UPDATE river_job SET state='running' WHERE id=?") + .bind(running) + .execute(&db.pool) + .await + .unwrap(); + let mut lock = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + sqlx::query("SELECT id FROM river_job WHERE id IN (?, ?) FOR UPDATE") + .bind(running) + .bind(locked) + .fetch_all(&mut *lock) + .await + .unwrap(); + let mut connection = db.pool.acquire().await.unwrap(); + let deleted = tokio::time::timeout( + Duration::from_secs(2), + cleanup_jobs( + &mut connection, + &CleanupJobs { + cancelled_before: None, + completed_before: Some(Timestamp::now()), + discarded_before: None, + limit: 1000, + queues_excluded: &[], + queues_included: None, + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deleted, 1000); + assert!(db.client.jobs().get(expired).await.is_err()); + assert_eq!( + db.client.jobs().get(locked).await.unwrap().state, + JobState::Completed + ); + assert_eq!( + db.client.jobs().get(running).await.unwrap().state, + JobState::Running + ); + drop(connection); + lock.rollback().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn committed_transactions_preserve_savepoints_and_reuse_connections() { + let db = TestDatabase::new().await; + let pool = MySqlPoolOptions::new() + .max_connections(1) + .connect_with((*db.pool.connect_options()).clone()) + .await + .unwrap(); + let mut transaction = Transaction::begin(&pool).await.unwrap(); + let connection_id: u64 = sqlx::query_scalar("SELECT CONNECTION_ID()") + .fetch_one(&mut *transaction) + .await + .unwrap(); + let mut savepoint = sqlx::Connection::begin(&mut *transaction).await.unwrap(); + let inserted = db + .client + .insert(Args { value: 1 }) + .tx(&mut savepoint) + .await + .unwrap(); + savepoint.rollback().await.unwrap(); + transaction.commit().await.unwrap(); + + let client = Client::builder(pool.clone()).build().unwrap(); + assert!( + client + .jobs() + .list(JobListParams::default()) + .await + .unwrap() + .jobs + .is_empty() + ); + assert!(client.jobs().get(inserted.job.row.id).await.is_err()); + assert_eq!( + client + .queues() + .list(crate::QueueListParams::default()) + .await + .unwrap(), + [] as [crate::Queue; 0] + ); + + let reused_connection_id: u64 = sqlx::query_scalar("SELECT CONNECTION_ID()") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(connection_id, reused_connection_id); + pool.close().await; + db.close().await; +} + +#[tokio::test] +async fn migrations_are_serialized_and_reversible() { + let db = TestDatabase::new().await; + let migrator = MySqlMigrator::new(db.pool.clone()); + assert!(migrator.validate(None).await.unwrap().is_valid()); + migrator + .migrate(Direction::Down, MigrateOpts::new().with_target_version(-1)) + .await + .unwrap(); + assert_eq!(migrator.existing_versions().await.unwrap(), [] as [i64; 0]); + let (left, right) = tokio::join!(migrator.migrate_up(), migrator.migrate_up()); + assert_eq!(left.unwrap().len() + right.unwrap().len(), 8); + assert_eq!( + migrator.existing_versions().await.unwrap(), + (1..=8).collect::>() + ); + db.close().await; +} + +#[tokio::test] +async fn insertion_and_completion_use_the_callers_transaction() { + let db = TestDatabase::new().await; + let now = "2026-01-02T03:04:05.123456Z".parse::().unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let inserted = db + .client + .insert(Args { + value: 9_007_199_254_740_993, + }) + .opts( + InsertOpts::default() + .with_max_attempts(40_000) + .with_scheduled_at(now), + ) + .tx(&mut tx) + .await + .unwrap(); + assert_eq!(inserted.job.args.value, 9_007_199_254_740_993); + assert_eq!(inserted.job.row.scheduled_at, now); + assert_eq!(inserted.job.row.max_attempts, 40_000); + tx.rollback().await.unwrap(); + assert!(db.client.jobs().get(inserted.job.row.id).await.is_err()); + + let inserted = db.client.insert(Args { value: 1 }).await.unwrap(); + let id = inserted.job.row.id; + sqlx::query("UPDATE river_job SET state = 'running', attempt = 1, attempted_at = UTC_TIMESTAMP(6) WHERE id = ?").bind(id).execute(&db.pool).await.unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + db.client.jobs().complete(id).tx(&mut tx).await.unwrap(); + tx.rollback().await.unwrap(); + assert_eq!( + db.client.jobs().get(id).await.unwrap().state, + JobState::Running + ); + db.close().await; +} + +#[tokio::test] +async fn concurrent_unique_insertion_returns_one_job() { + let db = TestDatabase::new().await; + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..12 { + let client = db.client.clone(); + tasks.spawn(async move { + client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_unique(UniqueOpts::new().with_by_args(true))) + .await + .unwrap() + }); + } + let mut ids = HashSet::new(); + let mut inserted = 0; + while let Some(result) = tasks.join_next().await { + let result = result.unwrap(); + ids.insert(result.job.row.id); + inserted += usize::from(!result.unique_skipped_as_duplicate); + } + assert_eq!(ids.len(), 1); + assert_eq!(inserted, 1); + db.close().await; +} + +#[tokio::test] +async fn concurrent_claims_do_not_duplicate_jobs_and_respect_paused_queues() { + let db = TestDatabase::new().await; + for value in 0..40 { + db.client.insert(Args { value }).await.unwrap(); + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + queue_upsert( + &mut tx, + "paused", + &Map::new(), + Some(Timestamp::now()), + Timestamp::now(), + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + db.client + .insert(Args { value: 0 }) + .opts(InsertOpts::default().with_queue("paused")) + .await + .unwrap(); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let pool = db.pool.clone(); + tasks.spawn(async move { + let mut tx = crate::database::begin_mysql_write(&pool).await.unwrap(); + let rows = claim( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 10, + max_attempted_by: 100, + now: Timestamp::now(), + queue: "default", + }, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + rows.into_iter() + .map(|row| row.unwrap().id) + .collect::>() + }); + } + let mut ids = HashSet::new(); + while let Some(result) = tasks.join_next().await { + for id in result.unwrap() { + assert!(ids.insert(id)); + } + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let params = ClaimJobs { + client_id: "worker", + kinds: None, + limit: 50, + max_attempted_by: 100, + now: Timestamp::now(), + queue: "default", + }; + for row in claim(&mut tx, ¶ms).await.unwrap() { + assert!(ids.insert(row.unwrap().id)); + } + assert_eq!(ids.len(), 40); + assert!( + claim( + &mut tx, + &ClaimJobs { + queue: "paused", + ..params + } + ) + .await + .unwrap() + .is_empty() + ); + tx.commit().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn cancellation_wins_over_retry_and_metadata_preserves_nulls() { + let db = TestDatabase::new().await; + let inserted = db.client.insert(Args { value: 1 }).await.unwrap(); + let id = inserted.job.row.id; + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + claim_selected( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 1, + max_attempted_by: 100, + now: Timestamp::now(), + queue: "default", + }, + &[id], + ) + .await + .unwrap(); + cancel(&mut tx, id, Timestamp::now()).await.unwrap(); + let metadata = Map::from_iter([("output".into(), Value::Null)]); + let row = complete( + &mut tx, + &CompleteJob { + attempt: Some(0), + error: None, + finalized_at: None, + id, + metadata_updates: Some(&metadata), + now: Timestamp::now(), + scheduled_at: Some(Timestamp::now()), + state: JobState::Available, + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(row.state, JobState::Cancelled); + assert_eq!(row.attempt, 1); + assert_eq!( + row.metadata.get::("output").unwrap(), + Some(Value::Null) + ); + assert!(row.finalized_at.is_some()); + let retried = retry(&mut tx, id, Timestamp::now()).await.unwrap().unwrap(); + assert_eq!(retried.state, JobState::Available); + assert!( + retried + .metadata + .get::("cancel_attempted_at") + .unwrap() + .is_none() + ); + assert_eq!( + retried.metadata.get::("output").unwrap(), + Some(Value::Null) + ); + tx.commit().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn metadata_containment_preserves_structure_and_exact_queue_names() { + let db = TestDatabase::new().await; + let wanted = db + .client + .insert(Args { value: 1 }) + .opts( + InsertOpts::default().with_metadata( + json!({"obj":{"a":1,"b":2},"array":[[1,2]],"nil":null}) + .as_object() + .unwrap() + .clone(), + ), + ) + .await + .unwrap(); + let other = db.client.insert(Args { value: 2 }).await.unwrap(); + sqlx::query("UPDATE river_job SET queue = 'default ' WHERE id = ?") + .bind(other.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + for (fragment, matches) in [ + (json!({"obj":{"a":1}}), true), + (json!({"obj":{"a":2}}), false), + (json!({"array":[[1]]}), true), + (json!({"array":[1]}), false), + (json!({"nil":null}), true), + (json!({"absent":null}), false), + ] { + let result = db + .client + .jobs() + .list(JobListParams::default().metadata(fragment.as_object().unwrap().clone())) + .await + .unwrap(); + assert_eq!( + result.jobs.iter().map(|job| job.id).collect::>(), + if matches { + vec![wanted.job.row.id] + } else { + vec![] + }, + "{fragment}" + ); + } + assert_eq!( + db.client + .jobs() + .list(JobListParams::default().queues(["default"])) + .await + .unwrap() + .jobs + .len(), + 1 + ); + db.close().await; +} + +#[tokio::test] +async fn outbox_waits_for_lower_uncommitted_ids() { + let db = TestDatabase::new().await; + let mut listener = NotificationListener::default(); + let mut connection = db.pool.acquire().await.unwrap(); + listener.connect(&mut connection).await.unwrap(); + listener + .listen(&mut connection, "river_leadership") + .await + .unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + notification_insert( + &mut tx, + &[NotificationInput { + topic: "river_leadership", + payload: "first", + }], + ) + .await + .unwrap(); + db.client.request_resign().await.unwrap(); + let read = tokio::spawn(async move { + let first = listener.next(&mut connection).await.unwrap().unwrap(); + let second = listener.next(&mut connection).await.unwrap().unwrap(); + (first, second) + }); + // Wait for the actual blocked scan, then verify it holds its locks in + // an explicit transaction instead of relying on autocommit semantics. + let autocommit = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let autocommit: Option = sqlx::query_scalar( + "SELECT t.AUTOCOMMIT FROM performance_schema.data_lock_waits w \ + JOIN performance_schema.data_locks l \ + ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE \ + JOIN performance_schema.events_transactions_current t \ + ON t.THREAD_ID=w.REQUESTING_THREAD_ID \ + WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'", + ) + .bind(&db.name) + .fetch_optional(&db.admin) + .await + .unwrap(); + if let Some(autocommit) = autocommit { + break autocommit; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!(autocommit, "NO"); + tx.commit().await.unwrap(); + let (first, second) = tokio::time::timeout(Duration::from_secs(5), read) + .await + .unwrap() + .unwrap(); + assert_eq!(first.payload, "first"); + assert_eq!( + serde_json::from_str::(&second.payload).unwrap()["action"], + "request_resign" + ); + assert!(first.id < second.id); + db.close().await; +} + +#[tokio::test] +#[expect( + clippy::too_many_lines, + reason = "Exercises the complete maintenance lifecycle in one transaction" +)] +async fn maintenance_schedules_rescues_and_cleans_jobs() { + let db = TestDatabase::new().await; + let inserted = db + .client + .insert(Args { value: 1 }) + .opts( + InsertOpts::default() + .with_scheduled_at(Timestamp::now() + jiff::SignedDuration::from_hours(1)), + ) + .await + .unwrap(); + let now = Timestamp::now() + jiff::SignedDuration::from_hours(2); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let leader = leader_elect(&mut tx, "leader", now, Duration::from_secs(30)) + .await + .unwrap() + .unwrap(); + assert!( + leader_elect(&mut tx, "other", now, Duration::from_secs(30)) + .await + .unwrap() + .is_none() + ); + assert!( + leader_reelect( + &mut tx, + "leader", + leader.elected_at, + now, + Duration::from_mins(1) + ) + .await + .unwrap() + .is_some() + ); + assert!( + !leader_resign(&mut tx, "other", leader.elected_at) + .await + .unwrap() + ); + assert!( + leader_resign(&mut tx, "leader", leader.elected_at) + .await + .unwrap() + ); + let candidates = schedule_candidates(&mut tx, now, 10).await.unwrap(); + assert_eq!(candidates.len(), 1); + schedule_set_available(&mut tx, &[candidates[0].id]) + .await + .unwrap(); + claim( + &mut tx, + &ClaimJobs { + client_id: "worker", + kinds: None, + limit: 10, + max_attempted_by: 100, + now, + queue: "default", + }, + ) + .await + .unwrap(); + let rescue_time = now + jiff::SignedDuration::from_hours(1); + let error = AttemptError { + at: rescue_time, + attempt: 1, + error: "rescued".into(), + trace: String::new(), + }; + let rescued = rescue( + &mut tx, + &RescueJob { + error: &error, + finalized_at: Some(rescue_time), + id: inserted.job.row.id, + scheduled_at: now, + state: JobState::Discarded, + stuck_horizon: rescue_time, + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(rescued.state, JobState::Discarded); + assert_eq!(rescued.errors.len(), 1); + assert_eq!( + cleanup_jobs( + &mut tx, + &CleanupJobs { + cancelled_before: None, + completed_before: None, + discarded_before: Some(rescue_time + jiff::SignedDuration::from_secs(1)), + limit: 10, + queues_excluded: &[], + queues_included: None + } + ) + .await + .unwrap(), + 1 + ); + tx.commit().await.unwrap(); + db.close().await; +} + +async fn wait_state(client: &Client, id: i64, state: JobState) -> JobRow { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let row = client.jobs().get(id).await.unwrap(); + if row.state == state { + return row; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("job {id} did not reach {state:?}")) +} + +#[tokio::test] +#[expect( + clippy::too_many_lines, + reason = "Exercises peer notifications and maintenance against one worker runtime" +)] +async fn workers_receive_peer_notifications_and_run_maintenance() { + use crate::{Job, MaintenanceConfig, QueueConfig, WorkContext, WorkOutcome, Workers}; + let db = TestDatabase::new().await; + // Use a fresh peer for each notification so insert cooldowns cannot suppress it. + // Make the job due before commit so client/server clock skew cannot consume + // the notification before the job is eligible for a claim. + let insert_available = async |value| { + let peer = Client::builder(db.pool.clone()).build().unwrap(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let inserted = peer.insert(Args { value }).tx(&mut tx).await.unwrap(); + sqlx::query("UPDATE river_job SET scheduled_at = ? WHERE id = ?") + .bind(mysql_time( + Timestamp::now() - jiff::SignedDuration::from_secs(1), + )) + .bind(inserted.job.row.id) + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + inserted + }; + let mut workers = Workers::new(); + workers + .add_fn(|context: WorkContext, job: Job| async move { + if job.args.value < 0 { + context.cancellation_token().cancelled().await; + return Ok::<_, crate::BoxError>(WorkOutcome::Cancel); + } + context.record_output(json!({"value": job.args.value}))?; + Ok(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .id("mysql-worker") + .workers(workers) + .queue( + "default", + QueueConfig::new(4) + .with_fetch_cooldown(Duration::from_millis(1)) + .with_fetch_poll_interval(Duration::from_mins(1)), + ) + .maintenance( + MaintenanceConfig::default() + .with_elect_interval(Duration::from_millis(50)) + .with_scheduler_interval(Duration::from_millis(50)), + ) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let inserted = insert_available(42).await; + let completed = wait_state(&db.client, inserted.job.row.id, JobState::Completed).await; + assert_eq!( + completed.metadata.get::("output").unwrap(), + Some(json!({"value":42})) + ); + + let cancellation = insert_available(-1).await; + wait_state(&db.client, cancellation.job.row.id, JobState::Running).await; + db.client + .jobs() + .cancel(cancellation.job.row.id) + .await + .unwrap(); + wait_state(&db.client, cancellation.job.row.id, JobState::Cancelled).await; + + let scheduled = db + .client + .insert(Args { value: 5 }) + .opts( + InsertOpts::default() + .with_scheduled_at(Timestamp::now() + jiff::SignedDuration::from_hours(1)), + ) + .await + .unwrap(); + sqlx::query( + "UPDATE river_job SET scheduled_at = UTC_TIMESTAMP(6) - INTERVAL 1 SECOND WHERE id = ?", + ) + .bind(scheduled.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + wait_state(&db.client, scheduled.job.row.id, JobState::Completed).await; + + let term: sqlx::types::Text = sqlx::query_scalar( + "SELECT CONCAT(elected_at, 'Z') FROM river_leader WHERE name = 'default'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + db.client.request_resign().await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let current: Option> = sqlx::query_scalar( + "SELECT CONCAT(elected_at, 'Z') FROM river_leader WHERE name = 'default'", + ) + .fetch_optional(&db.pool) + .await + .unwrap(); + if current != Some(term) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn shutdown_interrupts_a_blocked_notification_poll() { + use crate::{Job, QueueConfig, WorkContext, WorkOutcome, Workers}; + let db = TestDatabase::new().await; + let mut workers = Workers::new(); + workers + .add_fn(|_: WorkContext, _: Job| async { + Ok::<_, crate::BoxError>(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let mut tx = db.pool.begin().await.unwrap(); + sqlx::query("INSERT INTO river_notification (topic, payload) VALUES ('river_insert', '{}')") + .execute(&mut *tx) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE WHERE l.OBJECT_SCHEMA=? AND l.OBJECT_NAME='river_notification'") + .bind(&db.name).fetch_one(&db.admin).await.unwrap(); + if waiting > 0 { break; } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), run.stop()) + .await + .unwrap() + .unwrap(); + tx.rollback().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn queue_names_are_exact() { + let db = TestDatabase::new().await; + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let names = ["queue", "queue ", "QUEUE", "cafĂ©", "cafe"]; + for name in names { + let queue = queue_upsert(&mut tx, name, &Map::new(), None, Timestamp::now()) + .await + .unwrap(); + assert_eq!(queue.name, name); + } + for name in names { + assert_eq!(queue_get(&mut tx, name).await.unwrap().unwrap().name, name); + } + tx.rollback().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn unrelated_unique_constraints_preserve_the_native_error() { + let db = TestDatabase::new().await; + sqlx::query("CREATE UNIQUE INDEX extra_kind_unique ON river_job (kind)") + .execute(&db.pool) + .await + .unwrap(); + db.client.insert(Args { value: 1 }).await.unwrap(); + let error = db.client.insert(Args { value: 2 }).await.unwrap_err(); + let crate::Error::Database(sqlx::Error::Database(error)) = error else { + panic!("expected the native duplicate key error, got {error:?}"); + }; + assert!(error.is_unique_violation()); + assert!(error.message().contains("extra_kind_unique")); + assert_eq!(db.client.jobs().list(JobListParams::default()).await.unwrap().jobs.len(), 1); + db.close().await; +} diff --git a/rust/riverqueue/src/database/mysql/tests/conformance.rs b/rust/riverqueue/src/database/mysql/tests/conformance.rs new file mode 100644 index 000000000..76a34533f --- /dev/null +++ b/rust/riverqueue/src/database/mysql/tests/conformance.rs @@ -0,0 +1,354 @@ +//! Persist Go-generated protocol values through the real MySQL backend. + +use super::*; +use crate::{Job, QueueConfig, QueueUpdateParams, WorkContext, WorkOutcome, Workers}; + +#[derive(Deserialize)] +struct NotificationFixture { + name: String, + payload: Value, + topic: String, +} + +#[derive(Deserialize)] +struct Protocol { + attempt_error: AttemptError, + job_states: Vec, + metadata_keys: HashMap, + notifications: Vec, +} + +#[derive(Deserialize)] +struct SnoozeFixture { + expected_snoozes: i64, + metadata: JobMetadata, + name: String, +} + +#[derive(Deserialize)] +struct Snoozes { + snooze_counters: Vec, +} + +#[derive(Deserialize)] +struct StateFixture { + state: JobState, + unique_bit: u8, +} + +fn protocol() -> Protocol { + serde_json::from_str(&crate::conformance::read_fixture("protocol_values.json")).unwrap() +} + +#[tokio::test] +async fn emits_go_notification_payloads() { + let db = TestDatabase::new().await; + let mut connection = db.pool.acquire().await.unwrap(); + queue_upsert( + &mut connection, + "priority", + &Map::new(), + None, + Timestamp::now(), + ) + .await + .unwrap(); + + for fixture in protocol().notifications { + match fixture.name.as_str() { + "cancel" => { + let job = db + .client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_queue("priority")) + .await + .unwrap(); + sqlx::query("UPDATE river_job SET id=42, state='running' WHERE id=?") + .bind(job.job.row.id) + .execute(&db.pool) + .await + .unwrap(); + db.client.jobs().cancel(42).await.unwrap(); + } + "insert" => { + // A fresh client has no local insert-notification cooldown. + Client::builder(db.pool.clone()) + .build() + .unwrap() + .insert(Args { value: 2 }) + .opts(InsertOpts::default().with_queue("priority")) + .await + .unwrap(); + } + "metadata_changed" => { + db.client + .queues() + .update( + "priority", + QueueUpdateParams::new() + .metadata(Map::from_iter([("owner".into(), json!("candidate"))])), + ) + .await + .unwrap(); + } + "pause" => db.client.queues().pause("priority").await.unwrap(), + "request_resign" => db.client.request_resign().await.unwrap(), + "resigned" => { + // Like Go's MySQL/SQLite drivers, Rust does not emit a + // resignation notification. Receiving it is tested below. + let before = notification_last_id(&mut connection).await.unwrap(); + let leader = leader_elect( + &mut connection, + "client-1", + Timestamp::now(), + Duration::from_mins(1), + ) + .await + .unwrap() + .unwrap(); + assert!( + leader_resign(&mut connection, "client-1", leader.elected_at) + .await + .unwrap() + ); + assert_eq!(notification_last_id(&mut connection).await.unwrap(), before); + continue; + } + "resume" => db.client.queues().resume("priority").await.unwrap(), + name => panic!("unhandled Go notification {name}"), + } + let (topic, payload): (String, String) = sqlx::query_as( + "SELECT topic, CAST(payload AS CHAR) FROM river_notification ORDER BY id DESC LIMIT 1", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(topic, fixture.topic, "{}", fixture.name); + assert_eq!( + serde_json::from_str::(&payload).unwrap(), + fixture.payload, + "{}", + fixture.name + ); + } + drop(connection); + db.close().await; +} + +#[tokio::test] +async fn persists_go_snooze_counters_through_workers() { + let db = TestDatabase::new().await; + let fixture: Snoozes = + serde_json::from_str(&crate::conformance::read_fixture("snooze_counters.json")).unwrap(); + assert!(!fixture.snooze_counters.is_empty()); + let mut workers = Workers::new(); + workers + .add_fn(|_: WorkContext, _: Job| async { + Ok::<_, crate::BoxError>(WorkOutcome::Snooze(Duration::from_hours(1))) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + for case in fixture.snooze_counters { + let inserted = db + .client + .insert(Args { value: 1 }) + .opts(InsertOpts::default().with_metadata(case.metadata)) + .await + .unwrap(); + let stored = wait_state(&db.client, inserted.job.row.id, JobState::Scheduled).await; + assert_eq!(stored.attempt, 0, "{}", case.name); + assert_eq!( + stored.metadata.get::("snoozes").unwrap(), + Some(case.expected_snoozes), + "{}", + case.name + ); + } + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn receives_go_notifications_after_commit_and_ignores_rollback() { + let db = TestDatabase::new().await; + let mut connection = db.pool.acquire().await.unwrap(); + let mut listener = NotificationListener::default(); + listener.connect(&mut connection).await.unwrap(); + let fixture = protocol(); + for notification in &fixture.notifications { + listener + .listen(&mut connection, ¬ification.topic) + .await + .unwrap(); + } + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + for notification in &fixture.notifications { + sqlx::query("INSERT INTO river_notification(topic,payload) VALUES (?,?)") + .bind(¬ification.topic) + .bind(notification.payload.to_string()) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.rollback().await.unwrap(); + assert!(listener.next(&mut connection).await.unwrap().is_none()); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + for notification in &fixture.notifications { + sqlx::query("INSERT INTO river_notification(topic,payload) VALUES (?,?)") + .bind(¬ification.topic) + .bind(notification.payload.to_string()) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + for expected in fixture.notifications { + let received = listener.next(&mut connection).await.unwrap().unwrap(); + assert_eq!(received.topic, expected.topic); + assert_eq!( + serde_json::from_str::(&received.payload).unwrap(), + expected.payload + ); + } + assert!(listener.next(&mut connection).await.unwrap().is_none()); + drop(connection); + db.close().await; +} + +#[tokio::test] +async fn round_trips_go_errors_and_persists_rescue_and_output_metadata() { + let db = TestDatabase::new().await; + let fixture = protocol(); + let keys = &fixture.metadata_keys; + let inserted = db + .client + .insert(Args { value: 1 }) + .opts( + InsertOpts::default() + .with_metadata(Map::from_iter([(keys["rescue_count"].clone(), json!(2))])), + ) + .await + .unwrap(); + let id = inserted.job.row.id; + let nonce = inserted + .job + .row + .metadata + .get::(&keys["unique_nonce"]) + .unwrap() + .unwrap(); + assert_ne!(nonce, ""); + sqlx::query("UPDATE river_job SET state='running', attempt=3, attempted_at=UTC_TIMESTAMP(6), errors=JSON_ARRAY(CAST(? AS JSON)) WHERE id=?") + .bind(serde_json::to_string(&fixture.attempt_error).unwrap()).bind(id).execute(&db.pool).await.unwrap(); + assert_eq!( + db.client.jobs().get(id).await.unwrap().errors, + vec![fixture.attempt_error.clone()] + ); + let now = Timestamp::now(); + let mut tx = crate::database::begin_mysql_write(&db.pool).await.unwrap(); + let rescued = rescue( + &mut tx, + &RescueJob { + error: &fixture.attempt_error, + finalized_at: None, + id, + scheduled_at: now, + state: JobState::Retryable, + stuck_horizon: now + jiff::SignedDuration::from_hours(1), + }, + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + // MySQL JSON arithmetic returns a double, as in Go's driver. + rescued.metadata.get::(&keys["rescue_count"]).unwrap(), + Some(3.0) + ); + tx.commit().await.unwrap(); + let mut workers = Workers::new(); + workers + .add_fn(|context: WorkContext, _: Job| async move { + context.record_output(json!({"ok": true}))?; + Ok::<_, crate::BoxError>(WorkOutcome::Complete) + }) + .unwrap(); + let worker = Client::builder(db.pool.clone()) + .workers(workers) + .queue("default", QueueConfig::new(1)) + .build() + .unwrap(); + db.client.jobs().retry(id).await.unwrap(); + let mut run = worker.start().unwrap(); + run.wait_ready().await.unwrap(); + let stored = wait_state(&db.client, id, JobState::Completed).await; + assert_eq!( + stored.metadata.get::(&keys["output"]).unwrap(), + Some(json!({"ok": true})) + ); + assert_eq!( + stored.metadata.get::(&keys["rescue_count"]).unwrap(), + Some(3.0) + ); + assert_eq!( + stored + .metadata + .get::(&keys["unique_nonce"]) + .unwrap(), + Some(nonce) + ); + assert_eq!( + stored.errors, + vec![fixture.attempt_error.clone(), fixture.attempt_error] + ); + run.stop().await.unwrap(); + db.close().await; +} + +#[tokio::test] +async fn stored_states_and_unique_bits_match_go() { + let db = TestDatabase::new().await; + let opts = InsertOpts::default().with_unique( + UniqueOpts::new() + .with_by_args(true) + .with_by_state(JobState::ALL.iter().copied()), + ); + let inserted = db + .client + .insert(Args { value: 1 }) + .opts(opts.clone()) + .await + .unwrap(); + for fixture in protocol().job_states { + let finalized = matches!( + fixture.state, + JobState::Cancelled | JobState::Completed | JobState::Discarded + ); + sqlx::query("UPDATE river_job SET state=?, unique_states=?, finalized_at=IF(?, UTC_TIMESTAMP(6), NULL) WHERE id=?") + .bind(fixture.state.as_str()).bind(fixture.unique_bit).bind(finalized).bind(inserted.job.row.id).execute(&db.pool).await.unwrap(); + let stored = db.client.jobs().get(inserted.job.row.id).await.unwrap(); + assert_eq!(stored.state, fixture.state); + assert_eq!(stored.unique_states, Some(vec![fixture.state])); + let duplicate = db + .client + .insert(Args { value: 1 }) + .opts(opts.clone()) + .await + .unwrap(); + assert!(duplicate.unique_skipped_as_duplicate, "{}", fixture.state); + assert_eq!(duplicate.job.row.id, inserted.job.row.id); + let bit: i16 = sqlx::query_scalar("SELECT unique_states FROM river_job WHERE id=?") + .bind(inserted.job.row.id) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(bit, i16::from(fixture.unique_bit)); + } + db.close().await; +} diff --git a/rust/riverqueue/src/lib.rs b/rust/riverqueue/src/lib.rs index c627071f9..d606de74c 100644 --- a/rust/riverqueue/src/lib.rs +++ b/rust/riverqueue/src/lib.rs @@ -1,4 +1,4 @@ -// The README's examples use both backends, so it's the crate documentation +// The README's Postgres and SQLite examples need both features, so it's the crate documentation // (and its examples run as doctests) when both are enabled, as on docs.rs. #![cfg_attr( all(feature = "postgres", feature = "sqlite"), @@ -14,8 +14,10 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -#[cfg(not(any(feature = "postgres", feature = "sqlite")))] -compile_error!("riverqueue requires at least one database feature: `postgres` or `sqlite`"); +#[cfg(not(any(feature = "postgres", feature = "sqlite", feature = "mysql")))] +compile_error!( + "riverqueue requires at least one database feature: `mysql`, `postgres`, or `sqlite`" +); extern crate self as riverqueue; diff --git a/rust/riverqueue/src/maintenance/cleaner.rs b/rust/riverqueue/src/maintenance/cleaner.rs index 760c0164f..499661872 100644 --- a/rust/riverqueue/src/maintenance/cleaner.rs +++ b/rust/riverqueue/src/maintenance/cleaner.rs @@ -1,4 +1,4 @@ -//! Deletes expired finalized jobs, idle queues, and (on SQLite) delivered +//! Deletes expired finalized jobs, idle queues, and (on MySQL/SQLite) delivered //! notifications, ports of Go's `JobCleaner`, `QueueCleaner`, and //! `SQLiteNotificationCleaner`. @@ -15,6 +15,8 @@ use crate::__private::{ use crate::Error; use crate::client::ClientInner; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::pilot::job_delete_error; @@ -25,11 +27,11 @@ use super::{ }; /// Interval of the SQLite notification cleaner. -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] pub(super) const NOTIFICATION_CLEANER_INTERVAL: Duration = Duration::from_mins(1); /// Age after which SQLite notification outbox rows are deleted. -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] const NOTIFICATION_RETENTION: Duration = Duration::from_mins(5); fn horizon(now: Timestamp, retention: Option) -> Result, Error> { @@ -78,6 +80,21 @@ async fn clean_jobs_batch( ) -> Result { let timeout = context.inner.maintenance.job_cleaner_timeout; match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let operation = async { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let count = job_delete_before( + &context.inner, + DatabaseConnection::MySql(&mut transaction), + params, + ) + .await?; + transaction.commit().await?; + Ok::<_, MaintenanceError>(count) + }; + super::mysql_cancellable(&context.cancel, timeout, operation).await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let operation = async { @@ -165,6 +182,10 @@ pub(crate) async fn standard_job_delete_before( DatabaseConnection::Postgres(connection) => { Ok(postgres_delete_finalized_jobs(connection, &inner.schema, params).await?) } + #[cfg(feature = "mysql")] + DatabaseConnection::MySql(connection) => mysql_delete_finalized_jobs(connection, params) + .await + .map_err(|error| Error::Database(error.into())), #[cfg(feature = "sqlite")] DatabaseConnection::Sqlite(connection) => sqlite_delete_finalized_jobs(connection, params) .await @@ -219,6 +240,34 @@ pub(crate) async fn postgres_delete_finalized_jobs( Ok(result.rows_affected()) } +/// Runs the job cleaner's deletion on MySQL. +#[cfg(feature = "mysql")] +pub(crate) async fn mysql_delete_finalized_jobs( + connection: &mut sqlx::MySqlConnection, + params: &FinalizedJobDeleteParams, +) -> Result { + let queues_excluded = params + .queues_excluded + .iter() + .map(String::as_str) + .collect::>(); + let queues_included = params + .queues_included + .as_ref() + .map(|queues| queues.iter().map(String::as_str).collect::>()); + mysql::cleanup_jobs( + connection, + &mysql::CleanupJobs { + cancelled_before: params.cancelled_before, + completed_before: params.completed_before, + discarded_before: params.discarded_before, + limit: i32::try_from(params.limit).unwrap_or(i32::MAX), + queues_excluded: &queues_excluded, + queues_included: queues_included.as_deref(), + }, + ) + .await +} /// Runs the job cleaner's deletion on SQLite. #[cfg(feature = "sqlite")] pub(crate) async fn sqlite_delete_finalized_jobs( @@ -273,6 +322,21 @@ async fn clean_queues_batch( limit: i64, ) -> Result { match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let operation = async { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let deleted = mysql::queue_delete_expired( + &mut transaction, + updated_before, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await?; + transaction.commit().await?; + Ok::<_, MaintenanceError>(deleted.len()) + }; + super::mysql_cancellable(&context.cancel, TIMEOUT_DEFAULT, operation).await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let operation = async { @@ -317,6 +381,40 @@ async fn clean_queues_batch( } } +/// Deletes MySQL notification outbox rows old enough that every poller has +/// consumed them, oldest first, in batches like River Go's +/// `MySQLNotificationCleaner`. +/// +/// Each batch commits on its own, releasing its row locks before the next batch. The horizon is fixed for the pass +/// so rows expiring meanwhile don't extend it. Repeated timeouts switch to +/// the reduced batch size. +#[cfg(feature = "mysql")] +pub(super) async fn clean_mysql_notifications( + context: &ServiceContext, + pool: &sqlx::MySqlPool, +) -> Result { + let retention = jiff::SignedDuration::try_from(NOTIFICATION_RETENTION) + .map_err(|error| Error::configuration_context("maintenance", error.to_string()))?; + let created_before = Timestamp::now() - retention; + let mut deleted = 0; + loop { + let limit = batch_size(&context.breakers.notification_cleaner); + let operation = async { + let mut connection = mysql::Connection::acquire(pool).await?; + let count = mysql::notification_cleanup(&mut connection, created_before, limit).await?; + connection.release(); + Ok::<_, MaintenanceError>(count) + }; + let result = super::mysql_cancellable(&context.cancel, TIMEOUT_DEFAULT, operation).await; + record_batch(&context.breakers.notification_cleaner, &result); + let count = result?; + deleted += count; + if count < u64::try_from(limit).unwrap_or(u64::MAX) { + return Ok(deleted); + } + batch_backoff(&context.cancel).await?; + } +} /// Deletes SQLite notification outbox rows old enough that every poller has /// consumed them, oldest first, in batches like River Go's /// `SQLiteNotificationCleaner`. diff --git a/rust/riverqueue/src/maintenance/elector.rs b/rust/riverqueue/src/maintenance/elector.rs index 397e179ca..2d97d67eb 100644 --- a/rust/riverqueue/src/maintenance/elector.rs +++ b/rust/riverqueue/src/maintenance/elector.rs @@ -11,6 +11,8 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::{Error, client::ClientInner}; @@ -445,6 +447,19 @@ impl DatabaseLeaderStore { impl LeaderStore for DatabaseLeaderStore { async fn elect(&self, ttl: Duration) -> Result, Error> { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let now = Timestamp::now(); + mysql::leader_delete_expired(&mut transaction, now) + .await + .map_err(mysql_error)?; + let leader = mysql::leader_elect(&mut transaction, &self.inner.id, now, ttl) + .await + .map_err(mysql_error)?; + transaction.commit().await?; + Ok(leader.map(|leader| leader.elected_at)) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -489,6 +504,21 @@ impl LeaderStore for DatabaseLeaderStore { ttl: Duration, ) -> Result, Error> { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let leader = mysql::leader_reelect( + &mut transaction, + &self.inner.id, + elected_at, + Timestamp::now(), + ttl, + ) + .await + .map_err(mysql_error)?; + transaction.commit().await?; + Ok(leader.map(|leader| leader.elected_at)) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -526,6 +556,17 @@ impl LeaderStore for DatabaseLeaderStore { async fn resign(&self, elected_at: Timestamp) -> Result { match self.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let resigned = mysql::leader_resign(&mut transaction, &self.inner.id, elected_at) + .await + .map_err(mysql_error)?; + // Like Go's MySQL driver, a resignation writes no outbox + // row; other clients bid at their next election attempt. + transaction.commit().await?; + Ok(resigned) + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let mut transaction = crate::database::begin_sqlite_write(pool).await?; @@ -569,6 +610,10 @@ impl LeaderStore for DatabaseLeaderStore { } } +#[cfg(feature = "mysql")] +fn mysql_error(error: mysql::BackendError) -> Error { + Error::Database(error.into()) +} #[cfg(feature = "sqlite")] fn sqlite_error(error: sqlite::BackendError) -> Error { Error::Database(error.into()) diff --git a/rust/riverqueue/src/maintenance/maintainer.rs b/rust/riverqueue/src/maintenance/maintainer.rs index 39cbc4a76..c9cf3b5c9 100644 --- a/rust/riverqueue/src/maintenance/maintainer.rs +++ b/rust/riverqueue/src/maintenance/maintainer.rs @@ -144,6 +144,23 @@ async fn run_term( |context| Box::pin(async move { cleaner::clean_queues(&context).await }), )); match inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + let pool = pool.clone(); + services.spawn(run_periodically( + Arc::clone(&context), + "MySQL notification cleaner", + cleaner::NOTIFICATION_CLEANER_INTERVAL, + move |context| { + let pool = pool.clone(); + Box::pin(async move { + cleaner::clean_mysql_notifications(&context, &pool) + .await + .map(|_| ()) + }) + }, + )); + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { let pool = pool.clone(); diff --git a/rust/riverqueue/src/maintenance/mod.rs b/rust/riverqueue/src/maintenance/mod.rs index 7c21e501e..eaa4b0d5f 100644 --- a/rust/riverqueue/src/maintenance/mod.rs +++ b/rust/riverqueue/src/maintenance/mod.rs @@ -29,6 +29,8 @@ mod scheduler; #[cfg(all(test, feature = "postgres", river_postgres_tests))] mod tests; +#[cfg(feature = "mysql")] +pub(crate) use cleaner::mysql_delete_finalized_jobs; #[cfg(feature = "postgres")] pub(crate) use cleaner::postgres_delete_finalized_jobs; #[cfg(feature = "sqlite")] @@ -190,6 +192,12 @@ impl From for MaintenanceError { } } +#[cfg(feature = "mysql")] +impl From for MaintenanceError { + fn from(error: crate::database::mysql::BackendError) -> Self { + Self::Failed(Error::Database(error.into())) + } +} #[cfg(feature = "sqlite")] impl From for MaintenanceError { fn from(error: crate::database::sqlite::BackendError) -> Self { @@ -267,7 +275,7 @@ impl ReducedBatchBreaker { #[derive(Debug)] pub(crate) struct Breakers { pub(crate) job_cleaner: Mutex, - #[cfg(feature = "sqlite")] + #[cfg(any(feature = "sqlite", feature = "mysql"))] pub(crate) notification_cleaner: Mutex, pub(crate) queue_cleaner: Mutex, pub(crate) rescuer: Mutex, @@ -278,7 +286,7 @@ impl Breakers { fn new(sizes: BatchSizes) -> Self { Self { job_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), - #[cfg(feature = "sqlite")] + #[cfg(any(feature = "sqlite", feature = "mysql"))] notification_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), queue_cleaner: Mutex::new(ReducedBatchBreaker::new(sizes)), rescuer: Mutex::new(ReducedBatchBreaker::new(sizes)), @@ -361,6 +369,28 @@ async fn batch_backoff(cancel: &CancellationToken) -> Result<(), MaintenanceErro } } +/// Runs a MySQL operation under the term token and a client-side deadline. +/// +/// Dropping a batch also drops its transaction; SQLx rolls it back before +/// reusing the connection. +#[cfg(feature = "mysql")] +async fn mysql_cancellable( + cancel: &CancellationToken, + timeout: Duration, + operation: impl Future>, +) -> Result +where + MaintenanceError: From, +{ + tokio::select! { + biased; + () = cancel.cancelled() => Err(MaintenanceError::Cancelled), + result = tokio::time::timeout(timeout, operation) => match result { + Ok(result) => result.map_err(MaintenanceError::from), + Err(_) => Err(MaintenanceError::TimedOut), + }, + } +} /// Runs a SQLite operation under the term token and a client-side deadline. /// /// SQLite executes statements on the client, so dropping the future cannot diff --git a/rust/riverqueue/src/maintenance/rescuer.rs b/rust/riverqueue/src/maintenance/rescuer.rs index 0d550e78f..72e565e9a 100644 --- a/rust/riverqueue/src/maintenance/rescuer.rs +++ b/rust/riverqueue/src/maintenance/rescuer.rs @@ -13,6 +13,8 @@ use crate::__private::{ #[cfg(feature = "postgres")] use crate::client::{JobRecord, job_projection, tolerant_row}; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "sqlite")] use crate::database::sqlite; use crate::{AttemptError, Error, JobRow, JobState, WorkerTimeout, client::ClientInner}; @@ -64,6 +66,15 @@ async fn rescue_batch( stuck_horizon: Timestamp, ) -> Result { match context.inner.database.pool() { + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => { + super::mysql_cancellable( + &context.cancel, + TIMEOUT_DEFAULT, + rescue_batch_mysql(&context.inner, pool, after_id, limit, stuck_horizon), + ) + .await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { super::sqlite_cancellable( @@ -399,6 +410,92 @@ async fn rescue_batch_postgres( Ok(batch) } +#[cfg(feature = "mysql")] +async fn rescue_batch_mysql( + inner: &ClientInner, + pool: &sqlx::MySqlPool, + after_id: i64, + limit: i64, + stuck_horizon: Timestamp, +) -> Result { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let selected_ids = if inner.pilot.intercepts_rescue() { + inner + .pilot + .select_rescue_job_ids( + PilotDatabaseConnection::MySql(&mut transaction), + &rescue_params(inner, after_id, limit, stuck_horizon), + ) + .await + .map_err(|source| Error::Extension { + phase: crate::ExtensionPhase::AddOn { + operation: "rescue selection", + }, + source, + })? + } else { + None + }; + let rows = match selected_ids { + Some(ids) => mysql::jobs_by_ids(&mut transaction, &ids).await?, + None => { + mysql::stuck_jobs( + &mut transaction, + after_id, + stuck_horizon, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await? + } + }; + let batch = Batch { + last_id: rows.last().map(|row| row.id), + selected: rows.len(), + }; + let jobs = rescue_jobs(inner, &rows, Timestamp::now())?; + if !jobs.is_empty() { + let params = RescueManyParams { + database: inner.pilot_database_config(), + jobs, + stuck_horizon, + }; + let action = if inner.pilot.intercepts_rescue() { + inner + .pilot + .rescue_jobs(PilotDatabaseConnection::MySql(&mut transaction), ¶ms) + .await + .map_err(|source| Error::Extension { + phase: crate::ExtensionPhase::AddOn { + operation: "rescue", + }, + source, + })? + } else { + RescueAction::Continue + }; + if action == RescueAction::Continue { + for job in ¶ms.jobs { + let state = job.state; + let error: AttemptError = + serde_json::from_value(job.attempt_error.clone()).map_err(Error::from)?; + mysql::rescue( + &mut transaction, + &mysql::RescueJob { + error: &error, + finalized_at: job.finalized_at, + id: job.id, + scheduled_at: job.scheduled_at, + state, + stuck_horizon, + }, + ) + .await?; + } + } + } + transaction.commit().await?; + Ok(batch) +} #[cfg(feature = "sqlite")] async fn rescue_batch_sqlite( inner: &ClientInner, diff --git a/rust/riverqueue/src/maintenance/scheduler.rs b/rust/riverqueue/src/maintenance/scheduler.rs index 9d3424a0a..8a4ce3b67 100644 --- a/rust/riverqueue/src/maintenance/scheduler.rs +++ b/rust/riverqueue/src/maintenance/scheduler.rs @@ -7,9 +7,11 @@ use jiff::Timestamp; #[cfg(feature = "postgres")] use sqlx::{AssertSqlSafe, Row}; -#[cfg(feature = "sqlite")] +#[cfg(any(feature = "sqlite", feature = "mysql"))] use crate::client::InsertNotifyLimiter; use crate::database::DatabasePool; +#[cfg(feature = "mysql")] +use crate::database::mysql; #[cfg(feature = "postgres")] use crate::database::postgres_capabilities::CapabilitiesCache; #[cfg(feature = "sqlite")] @@ -53,6 +55,20 @@ async fn schedule_batch(context: &ServiceContext, limit: i64) -> Result { + super::mysql_cancellable( + &context.cancel, + TIMEOUT_DEFAULT, + schedule_batch_mysql( + pool, + &context.inner.insert_notify_limiter, + look_ahead, + limit, + ), + ) + .await + } #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { super::sqlite_cancellable( @@ -208,6 +224,67 @@ async fn delivers_notifications( .supports_listen_notify) } +#[cfg(feature = "mysql")] +async fn schedule_batch_mysql( + pool: &sqlx::MySqlPool, + notify_limiter: &InsertNotifyLimiter, + look_ahead: Timestamp, + limit: i64, +) -> Result { + let mut transaction = crate::database::mysql::Transaction::begin(pool).await?; + let candidates = mysql::schedule_candidates( + &mut transaction, + look_ahead, + i32::try_from(limit).unwrap_or(i32::MAX), + ) + .await?; + let count = candidates.len(); + let mut available_ids = Vec::new(); + let mut conflict_ids = Vec::new(); + let mut scheduled = Vec::new(); + for candidate in candidates { + let Some(unique_key) = candidate.unique_key.as_deref() else { + available_ids.push(candidate.id); + continue; + }; + if mysql::schedule_has_unique_collision(&mut transaction, candidate.id, unique_key).await? { + conflict_ids.push(candidate.id); + } else { + // Transition unique jobs one at a time so that a later duplicate + // in the same batch observes the earlier one as a collision. + let available = + mysql::schedule_set_available(&mut transaction, &[candidate.id]).await?; + scheduled.extend(available.into_iter().map(|job| Scheduled { + queue: job.queue, + scheduled_at: job.scheduled_at, + })); + } + } + if !available_ids.is_empty() { + let available = mysql::schedule_set_available(&mut transaction, &available_ids).await?; + scheduled.extend(available.into_iter().map(|job| Scheduled { + queue: job.queue, + scheduled_at: job.scheduled_at, + })); + } + if !conflict_ids.is_empty() { + mysql::schedule_discard_conflicts(&mut transaction, &conflict_ids, look_ahead).await?; + } + let notified = notified_queues(&scheduled); + for queue in notify_limiter.due(notified.iter().map(String::as_str)) { + let payload = crate::protocol::insert_notification_payload(queue); + mysql::notification_insert( + &mut transaction, + &[mysql::NotificationInput { + payload: &payload, + topic: crate::NOTIFICATION_TOPIC_INSERT, + }], + ) + .await?; + } + transaction.commit().await?; + Ok(count) +} #[cfg(feature = "sqlite")] async fn schedule_batch_sqlite( pool: &sqlx::SqlitePool, diff --git a/rust/riverqueue/src/pilot.rs b/rust/riverqueue/src/pilot.rs index 83d2ca0ef..2cffc807b 100644 --- a/rust/riverqueue/src/pilot.rs +++ b/rust/riverqueue/src/pilot.rs @@ -12,6 +12,7 @@ use serde_json::{Map, Value}; use sqlx::Postgres; #[cfg(feature = "sqlite")] use sqlx::Sqlite; +#[cfg(any(feature = "postgres", feature = "sqlite"))] use sqlx::Transaction; use tokio_util::sync::CancellationToken; @@ -52,6 +53,10 @@ impl PilotDatabase { DatabasePool::Postgres(pool) => { PilotTransactionInner::Postgres(crate::database::begin_postgres(pool).await?) } + #[cfg(feature = "mysql")] + DatabasePool::MySql(pool) => PilotTransactionInner::MySql( + crate::database::mysql::Transaction::begin(pool).await?, + ), #[cfg(feature = "sqlite")] DatabasePool::Sqlite(pool) => { PilotTransactionInner::Sqlite(crate::database::begin_sqlite_write(pool).await?) @@ -95,6 +100,8 @@ pub struct PilotTransaction(PilotTransactionInner); enum PilotTransactionInner { #[cfg(feature = "postgres")] Postgres(Transaction<'static, Postgres>), + #[cfg(feature = "mysql")] + MySql(crate::database::mysql::Transaction), #[cfg(feature = "sqlite")] Sqlite(Transaction<'static, Sqlite>), } @@ -109,6 +116,8 @@ impl PilotTransaction { match self.0 { #[cfg(feature = "postgres")] PilotTransactionInner::Postgres(transaction) => transaction.commit().await?, + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(transaction) => transaction.commit().await?, #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(transaction) => transaction.commit().await?, } @@ -122,6 +131,8 @@ impl PilotTransaction { PilotTransactionInner::Postgres(transaction) => { DatabaseConnection::Postgres(transaction) } + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(transaction) => DatabaseConnection::MySql(transaction), #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(transaction) => DatabaseConnection::Sqlite(transaction), } @@ -133,6 +144,8 @@ impl fmt::Debug for PilotTransaction { let kind = match &self.0 { #[cfg(feature = "postgres")] PilotTransactionInner::Postgres(_) => DatabaseKind::Postgres, + #[cfg(feature = "mysql")] + PilotTransactionInner::MySql(_) => DatabaseKind::MySql, #[cfg(feature = "sqlite")] PilotTransactionInner::Sqlite(_) => DatabaseKind::Sqlite, }; diff --git a/rust/riverqueue/src/storage.rs b/rust/riverqueue/src/storage.rs index a0f0851f4..24b4be493 100644 --- a/rust/riverqueue/src/storage.rs +++ b/rust/riverqueue/src/storage.rs @@ -11,6 +11,8 @@ //! it to the connection, [`AnyBackend`], and [`Session`] enums; the //! operations themselves don't change. +#[cfg(feature = "mysql")] +mod mysql; #[cfg(feature = "postgres")] mod postgres; #[cfg(feature = "sqlite")] @@ -21,6 +23,7 @@ use serde_json::{Map, Value}; use sqlx::Postgres; #[cfg(feature = "sqlite")] use sqlx::Sqlite; +#[cfg(any(feature = "postgres", feature = "sqlite"))] use sqlx::{Transaction, pool::PoolConnection}; use crate::__private::{ @@ -121,6 +124,8 @@ pub(crate) trait Backend { enum AnyBackend<'c> { #[cfg(feature = "postgres")] Postgres(postgres::PostgresBackend<'c>), + #[cfg(feature = "mysql")] + MySql(mysql::MySqlBackend<'c>), #[cfg(feature = "sqlite")] Sqlite(sqlite::SqliteBackend<'c>), } @@ -131,6 +136,8 @@ macro_rules! dispatch { match $any { #[cfg(feature = "postgres")] AnyBackend::Postgres($backend) => $call, + #[cfg(feature = "mysql")] + AnyBackend::MySql($backend) => $call, #[cfg(feature = "sqlite")] AnyBackend::Sqlite($backend) => $call, } @@ -241,6 +248,10 @@ impl<'c> Storage<'c> { schema: &inner.schema, }) } + #[cfg(feature = "mysql")] + DatabaseConnection::MySql(connection) => { + AnyBackend::MySql(mysql::MySqlBackend { connection }) + } #[cfg(feature = "sqlite")] DatabaseConnection::Sqlite(connection) => { AnyBackend::Sqlite(sqlite::SqliteBackend { connection }) @@ -527,8 +538,12 @@ enum SessionConnection<'a> { PostgresConnection(PoolConnection), #[cfg(feature = "postgres")] PostgresTransaction(Transaction<'static, Postgres>), + #[cfg(feature = "mysql")] + MySqlConnection(crate::database::mysql::Connection), #[cfg(feature = "sqlite")] SqliteConnection(PoolConnection), + #[cfg(feature = "mysql")] + MySqlTransaction(crate::database::mysql::Transaction), #[cfg(feature = "sqlite")] SqliteTransaction(Transaction<'static, Sqlite>), } @@ -552,10 +567,20 @@ impl<'a> Session<'a> { (DatabasePool::Postgres(pool), Access::Transaction) => { SessionConnection::PostgresTransaction(crate::database::begin_postgres(pool).await?) } + #[cfg(feature = "mysql")] + (DatabasePool::MySql(pool), Access::Autocommit) => SessionConnection::MySqlConnection( + crate::database::mysql::Connection::acquire(pool).await?, + ), #[cfg(feature = "sqlite")] (DatabasePool::Sqlite(pool), Access::Autocommit) => { SessionConnection::SqliteConnection(pool.acquire().await?) } + #[cfg(feature = "mysql")] + (DatabasePool::MySql(pool), Access::Transaction) => { + SessionConnection::MySqlTransaction( + crate::database::mysql::Transaction::begin(pool).await?, + ) + } #[cfg(feature = "sqlite")] (DatabasePool::Sqlite(pool), Access::Transaction) => { SessionConnection::SqliteTransaction( @@ -578,10 +603,16 @@ impl<'a> Session<'a> { SessionConnection::PostgresTransaction(transaction) => { DatabaseConnection::Postgres(transaction) } + #[cfg(feature = "mysql")] + SessionConnection::MySqlConnection(connection) => DatabaseConnection::MySql(connection), #[cfg(feature = "sqlite")] SessionConnection::SqliteConnection(connection) => { DatabaseConnection::Sqlite(connection) } + #[cfg(feature = "mysql")] + SessionConnection::MySqlTransaction(transaction) => { + DatabaseConnection::MySql(transaction) + } #[cfg(feature = "sqlite")] SessionConnection::SqliteTransaction(transaction) => { DatabaseConnection::Sqlite(transaction) @@ -590,19 +621,23 @@ impl<'a> Session<'a> { Storage::new(inner, connection) } - /// Commits a transaction River owns. A caller-managed transaction is - /// left for the caller to commit, and an autocommit connection has - /// nothing to commit. Dropping a session instead rolls River's own - /// transaction back. + /// Finishes the operation, committing River-owned transactions and returning + /// autocommit connections to their pools. Caller-managed transactions remain + /// uncommitted. Dropping a session instead rolls back River-owned transactions + /// and discards MySQL connections to avoid reusing an interrupted operation. pub(crate) async fn commit(self) -> Result<(), Error> { match self.connection { #[cfg(feature = "postgres")] SessionConnection::PostgresTransaction(transaction) => transaction.commit().await?, + #[cfg(feature = "mysql")] + SessionConnection::MySqlTransaction(transaction) => transaction.commit().await?, #[cfg(feature = "sqlite")] SessionConnection::SqliteTransaction(transaction) => transaction.commit().await?, SessionConnection::Caller(_) => {} #[cfg(feature = "postgres")] SessionConnection::PostgresConnection(_) => {} + #[cfg(feature = "mysql")] + SessionConnection::MySqlConnection(connection) => connection.release(), #[cfg(feature = "sqlite")] SessionConnection::SqliteConnection(_) => {} } @@ -635,8 +670,10 @@ pub(crate) async fn standard_job_delete_many( /// Creates the client's queue record or refreshes its `updated_at`. pub(crate) async fn touch_queue(inner: &ClientInner, name: &str) -> Result { - let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session.storage(inner).queue_touch(name).await + let mut session = Session::begin(&inner.database, Access::Transaction).await?; + let queue = session.storage(inner).queue_touch(name).await?; + session.commit().await?; + Ok(queue) } /// Returns the IDs among `ids` of running jobs with a cancellation request. @@ -645,17 +682,21 @@ pub(crate) async fn job_cancel_requested( ids: &[i64], ) -> Result, Error> { let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session + let requested = session .storage(inner) .backend .job_cancel_requested(ids) - .await + .await?; + session.commit().await?; + Ok(requested) } /// Loads a queue record, if one exists. pub(crate) async fn load_queue(inner: &ClientInner, name: &str) -> Result, Error> { let mut session = Session::begin(&inner.database, Access::Autocommit).await?; - session.storage(inner).queue_get(name).await + let queue = session.storage(inner).queue_get(name).await?; + session.commit().await?; + Ok(queue) } /// Transactional job operations for a [`WorkContext`](crate::WorkContext), diff --git a/rust/riverqueue/src/storage/mysql.rs b/rust/riverqueue/src/storage/mysql.rs new file mode 100644 index 000000000..c6dd02b22 --- /dev/null +++ b/rust/riverqueue/src/storage/mysql.rs @@ -0,0 +1,277 @@ +//! MySQL implementation of River's storage operations. + +use jiff::Timestamp; +use serde_json::{Map, Value}; +use sqlx::MySqlConnection; + +use super::Backend; +use crate::__private::DatabaseConnection; +use crate::database::mysql; +use crate::{Error, JobListParams, JobRow, JobState, Queue}; + +/// MySQL storage bound to one connection. +pub(super) struct MySqlBackend<'c> { + pub(super) connection: &'c mut MySqlConnection, +} + +impl MySqlBackend<'_> { + /// Lists jobs, optionally excluding running jobs before the limit + /// applies as bulk deletion does. + async fn list( + &mut self, + params: &JobListParams, + exclude_running: bool, + ) -> Result, Error> { + let kinds = params.kinds.iter().map(String::as_str).collect::>(); + let queues = params.queues.iter().map(String::as_str).collect::>(); + let tags_all = params + .tags_all + .iter() + .map(String::as_str) + .collect::>(); + let tags_any = params + .tags_any + .iter() + .map(String::as_str) + .collect::>(); + mysql::list( + self.connection, + &mysql::ListJobs { + exclude_running, + ids: ¶ms.ids, + keyset: params.keyset(), + kinds: &kinds, + limit: i32::try_from(params.limit).unwrap_or(i32::MAX), + metadata: params.metadata.as_ref(), + priorities: ¶ms.priorities, + queues: &queues, + states: ¶ms.states, + tags_all: &tags_all, + tags_any: &tags_any, + }, + ) + .await + .map_err(database_error) + } +} + +impl Backend for MySqlBackend<'_> { + fn connection(&mut self) -> DatabaseConnection<'_> { + DatabaseConnection::MySql(self.connection) + } + + async fn job_cancel(&mut self, id: i64) -> Result, Error> { + let updated = mysql::cancel(self.connection, id, Timestamp::now()) + .await + .map_err(database_error)?; + let Some(row) = updated else { + // Finalized jobs are returned unchanged, without a notification. + return self.job_get(id).await; + }; + let payload = serde_json::json!({ + "action": "cancel", + "job_id": id, + "queue": row.queue, + }) + .to_string(); + self.notify(crate::NOTIFICATION_TOPIC_CONTROL, &payload) + .await?; + Ok(Some(row)) + } + + // Like River Go, metadata that isn't valid JSON doesn't fail the lookup + // for the other jobs. + async fn job_cancel_requested(&mut self, ids: &[i64]) -> Result, Error> { + let ids = format!( + "[{}]", + ids.iter().map(i64::to_string).collect::>().join(",") + ); + Ok(sqlx::query_scalar( + "SELECT id FROM river_job \ + WHERE id IN (SELECT id FROM JSON_TABLE(?, '$[*]' COLUMNS(id BIGINT PATH '$')) AS wanted) \ + AND JSON_CONTAINS_PATH(metadata, 'one', '$.cancel_attempted_at') \ + AND state = 'running' \ + ORDER BY id", + ) + .bind(ids) + .fetch_all(&mut *self.connection) + .await?) + } + + async fn job_complete( + &mut self, + id: i64, + metadata_updates: &Map, + ) -> Result { + let Some(job) = mysql::get_locked(self.connection, id) + .await + .map_err(database_error)? + else { + return Err(Error::NotFound(crate::Record::Job(id))); + }; + if job.state != JobState::Running { + return Err(super::job_not_running(job.state)); + } + let now = Timestamp::now(); + mysql::complete( + self.connection, + &mysql::CompleteJob { + attempt: None, + error: None, + finalized_at: Some(now), + id, + metadata_updates: Some(metadata_updates), + now, + scheduled_at: None, + state: JobState::Completed, + }, + ) + .await + .map_err(database_error)? + .ok_or(Error::NotFound(crate::Record::Job(id))) + } + + async fn job_delete(&mut self, id: i64) -> Result { + if let Some(row) = mysql::delete(self.connection, id) + .await + .map_err(database_error)? + { + return Ok(row); + } + match mysql::get(self.connection, id) + .await + .map_err(database_error)? + { + Some(job) if job.state == JobState::Running => Err(Error::JobRunning), + None | Some(_) => Err(Error::NotFound(crate::Record::Job(id))), + } + } + + async fn job_delete_many(&mut self, filter: &JobListParams) -> Result, Error> { + // Running jobs are excluded before the limit applies, like Go. + // Each delete locks and rechecks the candidate's state. + let jobs = self.list(filter, true).await?; + let mut deleted = Vec::with_capacity(jobs.len()); + for job in jobs { + if let Some(row) = mysql::delete(self.connection, job.id) + .await + .map_err(database_error)? + { + deleted.push(row); + } + } + Ok(deleted) + } + + async fn job_claim( + &mut self, + id: i64, + client_id: &str, + max_attempted_by: i32, + ) -> Result, Error> { + let claimed = mysql::claim_selected( + self.connection, + &mysql::ClaimJobs { + client_id, + kinds: None, + limit: 1, + max_attempted_by, + now: Timestamp::now(), + queue: "", + }, + &[id], + ) + .await + .map_err(database_error)?; + Ok(claimed + .into_iter() + .next() + .and_then(crate::client::tolerant_row)) + } + + async fn job_get(&mut self, id: i64) -> Result, Error> { + mysql::get(self.connection, id) + .await + .map_err(database_error) + } + + async fn job_list(&mut self, params: &JobListParams) -> Result, Error> { + self.list(params, false).await + } + + async fn job_retry(&mut self, id: i64) -> Result, Error> { + let updated = mysql::retry(self.connection, id, Timestamp::now()) + .await + .map_err(database_error)?; + let Some(row) = updated else { + // Running and already-available jobs are returned unchanged. + return self.job_get(id).await; + }; + // Like Go's `JobRetry`, a retry sends no insert notification; + // producers find the job on their next poll. + Ok(Some(row)) + } + + async fn job_update( + &mut self, + id: i64, + metadata: &Map, + ) -> Result, Error> { + mysql::update(self.connection, id, metadata) + .await + .map_err(database_error) + } + + async fn notify(&mut self, topic: &str, payload: &str) -> Result<(), Error> { + mysql::notification_insert( + self.connection, + &[mysql::NotificationInput { payload, topic }], + ) + .await + .map_err(database_error)?; + Ok(()) + } + + async fn queue_get(&mut self, name: &str) -> Result, Error> { + mysql::queue_get(self.connection, name) + .await + .map_err(database_error) + } + + async fn queue_list(&mut self, limit: u32) -> Result, Error> { + mysql::queue_list(self.connection, i32::try_from(limit).unwrap_or(i32::MAX)) + .await + .map_err(database_error) + } + + async fn queue_set_paused(&mut self, name: &str, paused: bool) -> Result { + let now = Timestamp::now(); + let updated = if paused { + mysql::queue_pause(self.connection, name, now).await + } else { + mysql::queue_resume(self.connection, name, now).await + } + .map_err(database_error)?; + Ok(u64::try_from(updated.len()).unwrap_or(u64::MAX)) + } + + async fn queue_touch(&mut self, name: &str) -> Result { + mysql::queue_upsert(self.connection, name, &Map::new(), None, Timestamp::now()) + .await + .map_err(database_error) + } + + async fn queue_update( + &mut self, + name: &str, + metadata: Option<&Map>, + ) -> Result, Error> { + mysql::queue_update(self.connection, name, metadata, Timestamp::now()) + .await + .map_err(database_error) + } +} + +fn database_error(error: mysql::BackendError) -> Error { + Error::Database(error.into()) +} diff --git a/rust/riverqueue/tests/database_contract.rs b/rust/riverqueue/tests/database_contract.rs index b5ba2d2e1..dd2c264e5 100644 --- a/rust/riverqueue/tests/database_contract.rs +++ b/rust/riverqueue/tests/database_contract.rs @@ -28,6 +28,29 @@ fn sqlite_pool() -> SqlitePool { SqlitePoolOptions::new().connect_lazy_with(SqliteConnectOptions::new()) } +#[cfg(feature = "mysql")] +fn mysql_pool() -> sqlx::MySqlPool { + sqlx::mysql::MySqlPoolOptions::new().connect_lazy_with(sqlx::mysql::MySqlConnectOptions::new()) +} + +#[tokio::test] +#[cfg(feature = "mysql")] +async fn mysql_sources_and_transactions_implement_the_database_contract() { + use riverqueue::database::MySqlDatabase; + fn mysql_transaction(database: &Database, transaction: &mut Transaction<'_, sqlx::MySql>) { + assert_eq!(require_executor(database, transaction), DatabaseKind::MySql); + } + let _: for<'executor, 'transaction> fn( + &Database, + &'executor mut Transaction<'transaction, sqlx::MySql>, + ) = mysql_transaction; + let pool = mysql_pool(); + let database = require_source(MySqlDatabase::new(pool.clone())); + assert_eq!(database.kind(), DatabaseKind::MySql); + assert_eq!(database.postgres_schema(), None); + assert_eq!(require_source(pool).kind(), DatabaseKind::MySql); +} + fn require_executor<'executor, E: DatabaseTransactionExecutor<'executor>>( database: &Database, executor: E, From 585a746104d343047d93ef15dd5bd003bd6cae43 Mon Sep 17 00:00:00 2001 From: Brandur Date: Sun, 8 Nov 2026 11:55:54 -0600 Subject: [PATCH 2/2] [rust] Format MySQL implementation --- rust/riverqueue-migrate/src/mysql.rs | 17 +++++++----- rust/riverqueue/src/__private.rs | 4 ++- rust/riverqueue/src/database/mysql.rs | 29 +++++++++++++++------ rust/riverqueue/src/database/mysql/tests.rs | 11 +++++++- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/rust/riverqueue-migrate/src/mysql.rs b/rust/riverqueue-migrate/src/mysql.rs index a9d4bebe9..4379d3ea5 100644 --- a/rust/riverqueue-migrate/src/mysql.rs +++ b/rust/riverqueue-migrate/src/mysql.rs @@ -130,10 +130,11 @@ impl MySqlMigrator { opts: MigrateOpts, ) -> Result { let mut connection = self.pool.acquire().await?; - let acquired: Option = - sqlx::query_scalar("SELECT GET_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)), 10)") - .fetch_one(&mut *connection) - .await?; + let acquired: Option = sqlx::query_scalar( + "SELECT GET_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)), 10)", + ) + .fetch_one(&mut *connection) + .await?; if acquired != Some(1) { return Err(sqlx::Error::Protocol( "could not acquire MySQL migration lock; select a database in the connection URL" @@ -144,9 +145,11 @@ impl MySqlMigrator { // A task abort or panic closes the connection, releasing its named lock. connection.close_on_drop(); let result = migrate_on(&mut connection, direction, opts).await; - let released = sqlx::query("SELECT RELEASE_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)))") - .execute(&mut *connection) - .await; + let released = sqlx::query( + "SELECT RELEASE_LOCK(CONCAT('river_migrate:', LEFT(SHA2(DATABASE(), 256), 48)))", + ) + .execute(&mut *connection) + .await; match result { Ok(result) => { released?; diff --git a/rust/riverqueue/src/__private.rs b/rust/riverqueue/src/__private.rs index 2490afa30..48cc138f0 100644 --- a/rust/riverqueue/src/__private.rs +++ b/rust/riverqueue/src/__private.rs @@ -26,7 +26,9 @@ use tokio_util::sync::CancellationToken; pub use crate::client::{ExtensionClient, PreparedInsertRequest, RawInsertRequest, WeakClient}; pub use crate::database::erased::{Database, ErasedExecutor, ErasedTransaction}; #[cfg(feature = "mysql")] -pub use crate::database::mysql::{Connection as MySqlConnectionGuard, Transaction as MySqlTransactionGuard}; +pub use crate::database::mysql::{ + Connection as MySqlConnectionGuard, Transaction as MySqlTransactionGuard, +}; pub use crate::pilot::{ JobDeleteBeforeContext, JobDeleteBeforeNext, JobDeleteContext, JobDeleteManyContext, JobDeleteManyNext, JobDeleteNext, PilotDatabase, PilotProducer, PilotTransaction, diff --git a/rust/riverqueue/src/database/mysql.rs b/rust/riverqueue/src/database/mysql.rs index 896fea839..9ab2600ce 100644 --- a/rust/riverqueue/src/database/mysql.rs +++ b/rust/riverqueue/src/database/mysql.rs @@ -466,14 +466,27 @@ async fn insert_row( sql.push_str(" ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id)"); } sqlx::query(AssertSqlSafe(sql)) - .bind(params.id).bind(params.encoded_args.get()).bind(params.attempt) - .bind(mysql_time_optional(params.attempted_at)).bind(serde_json::to_string(params.attempted_by)?) - .bind(mysql_time_optional(params.created_at)).bind(serde_json::to_string(params.errors)?) - .bind(mysql_time_optional(params.finalized_at)).bind(params.kind).bind(params.max_attempts) - .bind(serde_json::to_string(&metadata)?).bind(params.priority).bind(params.queue) - .bind(mysql_time_optional(params.scheduled_at)).bind(params.state.as_str()) - .bind(serde_json::to_string(params.tags)?).bind(params.unique_key).bind(params.unique_states.map(i16::from)) - .execute(connection).await.map_err(Into::into) + .bind(params.id) + .bind(params.encoded_args.get()) + .bind(params.attempt) + .bind(mysql_time_optional(params.attempted_at)) + .bind(serde_json::to_string(params.attempted_by)?) + .bind(mysql_time_optional(params.created_at)) + .bind(serde_json::to_string(params.errors)?) + .bind(mysql_time_optional(params.finalized_at)) + .bind(params.kind) + .bind(params.max_attempts) + .bind(serde_json::to_string(&metadata)?) + .bind(params.priority) + .bind(params.queue) + .bind(mysql_time_optional(params.scheduled_at)) + .bind(params.state.as_str()) + .bind(serde_json::to_string(params.tags)?) + .bind(params.unique_key) + .bind(params.unique_states.map(i16::from)) + .execute(connection) + .await + .map_err(Into::into) } pub(crate) async fn claim( diff --git a/rust/riverqueue/src/database/mysql/tests.rs b/rust/riverqueue/src/database/mysql/tests.rs index 7a88da865..087b4a7c2 100644 --- a/rust/riverqueue/src/database/mysql/tests.rs +++ b/rust/riverqueue/src/database/mysql/tests.rs @@ -923,6 +923,15 @@ async fn unrelated_unique_constraints_preserve_the_native_error() { }; assert!(error.is_unique_violation()); assert!(error.message().contains("extra_kind_unique")); - assert_eq!(db.client.jobs().list(JobListParams::default()).await.unwrap().jobs.len(), 1); + assert_eq!( + db.client + .jobs() + .list(JobListParams::default()) + .await + .unwrap() + .jobs + .len(), + 1 + ); db.close().await; }