From 0f743b20cd24fd024cad7536e5f7bde933a20e66 Mon Sep 17 00:00:00 2001 From: Christopher Vollick <0@psycoti.ca> Date: Tue, 10 Jun 2025 15:47:40 -0400 Subject: [PATCH] Mark Outgoing Bitcoin Transactions as Ignored We have a bunch of transactions that get added to the queue every so often that we know we're never going to care about, but every time we re-assert to electrum we have to go back and hammer it about them anyway over and over again. So with this, we take the ones we know we're never going to care about and cache that we don't care in redis, so electrum can stop hearing about it. And it hopefully can read those answers somewhat quickly per-chunk so the entire bin can clear itself out more easily, all without having to bug our special baby! --- bin/process_pending_btc_transactions | 3 +- lib/pending_transaction_repo.rb | 42 +++++++++++++++++++++- test/test_pending_transaction_repo.rb | 50 +++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/bin/process_pending_btc_transactions b/bin/process_pending_btc_transactions index a7df1391ec0b4679674771d3ed856d24352ce177..79303a65ca136151e034073f3ad851ae7839d818 100755 --- a/bin/process_pending_btc_transactions +++ b/bin/process_pending_btc_transactions @@ -220,6 +220,7 @@ end repo = PendingTransactionRepo.new( "pending_#{CONFIG[:electrum][:currency]}_transactions", + ignored_key: "ignored_#{CONFIG[:electrum][:currency]}_transactions", customer_address_template: lambda { |customer_id| "jmp_customer_#{CONFIG[:electrum][:currency]}_addresses-#{customer_id}" } @@ -237,7 +238,7 @@ done = repo.map { |pending, customer_id| next unless pending.confirmations >= CONFIG[:required_confirmations] if pending.outgoing? - # This is a send, not a receive, do not record it + repo.mark_ignored(pending) repo.remove_transaction(pending) next end diff --git a/lib/pending_transaction_repo.rb b/lib/pending_transaction_repo.rb index 74ed59c03f02c25b37934e82728909a8bbaaa844..82c24995a4ddb6ec7ef7c9c3d00db5561d5032b1 100644 --- a/lib/pending_transaction_repo.rb +++ b/lib/pending_transaction_repo.rb @@ -26,9 +26,10 @@ class PendingTransactionRepo end end - def initialize(key, customer_address_template: nil) + def initialize(key, customer_address_template: nil, ignored_key: nil) @key = key @customer_address_template = customer_address_template + @ignored_key = ignored_key # Default handler allows all exceptions to be rethrown @error_handler = ->(*){} @@ -83,6 +84,32 @@ class PendingTransactionRepo end end + class IgnoredTransactionFilter + def initialize(redis, key) + @redis = redis + @key = key + end + + def filter_chunk(txids_and_customer_ids) + # There's an smismember command that was added to the rubygem in + # v4.5 but prod is currently v4.2.5 and I don't want to have to + # deal with that. So for now this sucks a little, using pipelining + # instead. + # Better than N full round-trip requests, but not as good as + # smismember probably + results = @redis.pipelined { + txids_and_customer_ids.each do |(txid, _customer_id)| + @redis.sismember(@key, txid) + end + } + + # Now filter by the results correspondingly + # Because it's in order, we pull each true or false off results and + # if it's true then we don't need to process the item further + txids_and_customer_ids.reject { |_v| results.shift } + end + end + class WrongCustomerFilter def initialize(redis, template) @redis = redis @@ -111,6 +138,7 @@ class PendingTransactionRepo def filter(txids_and_customer_ids) @filters ||= [ + IgnoredTransactionFilter.new(redis, @ignored_key), WrongCustomerFilter.new(redis, @customer_address_template), ExistingTransactionFilter.new(database) ] @@ -144,4 +172,16 @@ class PendingTransactionRepo def remove_transaction(pending) redis.hdel(@key, pending.txid) end + + # We use this to record transactions we've decided not to care about ever + # again. These get filtered out during the processing _before_ it gets to + # electrum or the user code, so effectively they just don't exist anymore. + # + # If electrum was more efficient this would be unnecessary because we'd + # just ask electrum about the thing and we could decide we don't care + # organically. But since it's a bit sensitive, this effectively serves as a + # cache of "things we'd end up ignoring anyway" + def mark_ignored(pending) + redis.sadd(@ignored_key, pending.txid) + end end diff --git a/test/test_pending_transaction_repo.rb b/test/test_pending_transaction_repo.rb index 69d4ae3b8c50f14b1574ea1d57af8c1b57391434..c15bb80d904ce455a2ce72c956706e2d7ff00210 100644 --- a/test/test_pending_transaction_repo.rb +++ b/test/test_pending_transaction_repo.rb @@ -180,6 +180,23 @@ class TestPendingTransactionRepo < Minitest::Test assert_mock repo.electrum end + def test_mark_ignored + repo = PendingTransactionRepo.new("key", ignored_key: "ig") + repo.setup_mocks + + pending = PendingTransactionRepo::PendingTransaction.new( + FakeElectrumTransaction.new("tx", 6, 0.5), + "addr" + ) + + repo.redis.expect(:sadd, nil, ["ig", "tx/addr"]) + + repo.mark_ignored(pending) + + assert_mock repo.redis + assert_mock repo.electrum + end + def test_chunking repo = PendingTransactionRepo.new("key") repo.setup_mocks @@ -273,6 +290,39 @@ class TestPendingTransactionRepo < Minitest::Test assert_mock db_mock end + def test_ignored_transaction_filter + redis = Object.new + + def redis.pipelined + @stuff = [] + yield + @stuff + end + + def redis.sismember(key, txid) + raise unless key == "key" + + @stuff << ["two/a", "three/b"].include?(txid) + end + + filter = PendingTransactionRepo::IgnoredTransactionFilter.new( + redis, "key" + ) + + remaining = filter.filter_chunk([ + ["one/a", "1234"], + ["two/a", "1234"], + ["three/b", "4321"], + ["four/c", "2323"] + ]) + + assert_equal( + [["one/a", "1234"], ["four/c", "2323"]], + remaining, + "should only include unfiltered results" + ) + end + def test_wrong_customer_filter redis = Object.new