sql: fix BROKEN for large channelmoves tables.

plugins/sql.c

@@ -162,6 +162,7 @@ struct sql {
 	int gosstore_fd ;
 	size_t gosstore_nodes_off, gosstore_channels_off;
 	u64 next_rowid;
+	u32 limit_per_list;
 
 	/* This is an aux_command for all our watches */
 	struct command *waitcmd;
@@ -1715,17 +1716,17 @@ static const char *db_table_name(const tal_t *ctx, const char *cmdname)
 	return ret;
 }
 
-#define LIMIT_PER_LIST 10000
-
 static struct command_result *limited_list_done(struct command *cmd,
 						const char *method,
 						const char *buf,
 						const jsmntok_t *result,
 						struct db_query *dbq)
 {
+	struct sql *sql = sql_of(dbq->cmd->plugin);
 	struct table_desc *td = dbq->tables[0];
 	struct command_result *ret;
 	size_t num_entries;
+	bool was_limited;
 
 	ret = process_json_result(cmd, buf, result, td, false,
 				  &td->last_created_index,
@@ -1734,27 +1735,30 @@ static struct command_result *limited_list_done(struct command *cmd,
 	if (ret)
 		return ret;
 
+	was_limited = num_entries == sql->limit_per_list;
+	if (!was_limited)
+		td->refresh_needs = REFRESH_UNNECESSARY;
 	/* If we got the number we asked for, we need to ask again. */
-	return one_refresh_done(cmd, dbq, num_entries == LIMIT_PER_LIST);
+	return one_refresh_done(cmd, dbq, was_limited);
 }
 
 /* The simplest case: append-only lists */
 static struct command_result *refresh_by_created_index(struct command *cmd,
 						       struct table_desc *td,
 						       struct db_query *dbq)
 {
+	struct sql *sql = sql_of(dbq->cmd->plugin);
 	struct out_req *req;
 
 	/* Since we're relying on watches, mark refreshing unnecessary to start */
 	assert(td->refresh_needs != REFRESH_UNNECESSARY);
-	td->refresh_needs = REFRESH_UNNECESSARY;
 
 	req = jsonrpc_request_start(cmd, td->cmdname,
 				    limited_list_done, forward_error,
 				    dbq);
 	json_add_string(req->js, "index", "created");
 	json_add_u64(req->js, "start", td->last_created_index + 1);
-	json_add_u64(req->js, "limit", LIMIT_PER_LIST);
+	json_add_u64(req->js, "limit", sql->limit_per_list);
 	return send_outreq(req);
 }
 
@@ -2216,11 +2220,16 @@ int main(int argc, char *argv[])
 	sql->gosstore_fd = -1;
 	sql->gosstore_nodes_off = sql->gosstore_channels_off = 0;
 	sql->next_rowid = 1;
+	sql->limit_per_list = 10000;
 	plugin_main(argv, init, take(sql), PLUGIN_RESTARTABLE, true, NULL, commands, ARRAY_SIZE(commands),
 	            NULL, 0, NULL, 0, NULL, 0,
 		    plugin_option_dev("dev-sqlfilename",
 				      "string",
 				      "Use on-disk sqlite3 file instead of in memory (e.g. debugging)",
 				      charp_option, NULL, &sql->dbfilename),
+		    plugin_option_dev("dev-sqllistlimit",
+				      "int",
+				      "A variable setting for how many chainmoves/channelmoves we fetch at once",
+				      u32_option, NULL, &sql->limit_per_list),
 		    NULL);
 }

tests/test_plugin.py

@@ -4410,6 +4410,16 @@ def test_sql_deprecated(node_factory, bitcoind):
     assert ret == {'rows': [[1]]}
 
 
+def test_sql_limit_per_list(node_factory):
+    l1, l2, l3 = node_factory.line_graph(
+        3, wait_for_announce=True, opts=[{}, {"dev-sqllistlimit": 10}, {}]
+    )
+    for i in range(20):
+        inv = l3.rpc.invoice(1000, f"inv-{i}", f"inv-{i}")["bolt11"]
+        l1.rpc.xpay(inv)
+    l2.rpc.sql("SELECT created_index, payment_hash FROM channelmoves")
+
+
 def test_plugin_persist_option(node_factory):
     """test that options from config file get remembered across plugin stop/start"""
     plugin_path = os.path.join(os.getcwd(), 'contrib/plugins/helloworld.py')