Pipeline.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. const calculateSlot = require("cluster-key-slot");
  4. const commands_1 = require("@ioredis/commands");
  5. const standard_as_callback_1 = require("standard-as-callback");
  6. const util_1 = require("util");
  7. const Command_1 = require("./Command");
  8. const utils_1 = require("./utils");
  9. const Commander_1 = require("./utils/Commander");
  10. /*
  11. This function derives from the cluster-key-slot implementation.
  12. Instead of checking that all keys have the same slot, it checks that all slots are served by the same set of nodes.
  13. If this is satisfied, it returns the first key's slot.
  14. */
  15. function generateMultiWithNodes(redis, keys) {
  16. const slot = calculateSlot(keys[0]);
  17. const target = redis._groupsBySlot[slot];
  18. for (let i = 1; i < keys.length; i++) {
  19. if (redis._groupsBySlot[calculateSlot(keys[i])] !== target) {
  20. return -1;
  21. }
  22. }
  23. return slot;
  24. }
  25. class Pipeline extends Commander_1.default {
  26. constructor(redis) {
  27. super();
  28. this.redis = redis;
  29. this.isPipeline = true;
  30. this.replyPending = 0;
  31. this._queue = [];
  32. this._result = [];
  33. this._transactions = 0;
  34. this._shaToScript = {};
  35. this.isCluster =
  36. this.redis.constructor.name === "Cluster" || this.redis.isCluster;
  37. this.options = redis.options;
  38. Object.keys(redis.scriptsSet).forEach((name) => {
  39. const script = redis.scriptsSet[name];
  40. this._shaToScript[script.sha] = script;
  41. this[name] = redis[name];
  42. this[name + "Buffer"] = redis[name + "Buffer"];
  43. });
  44. redis.addedBuiltinSet.forEach((name) => {
  45. this[name] = redis[name];
  46. this[name + "Buffer"] = redis[name + "Buffer"];
  47. });
  48. this.promise = new Promise((resolve, reject) => {
  49. this.resolve = resolve;
  50. this.reject = reject;
  51. });
  52. const _this = this;
  53. Object.defineProperty(this, "length", {
  54. get: function () {
  55. return _this._queue.length;
  56. },
  57. });
  58. }
  59. fillResult(value, position) {
  60. if (this._queue[position].name === "exec" && Array.isArray(value[1])) {
  61. const execLength = value[1].length;
  62. for (let i = 0; i < execLength; i++) {
  63. if (value[1][i] instanceof Error) {
  64. continue;
  65. }
  66. const cmd = this._queue[position - (execLength - i)];
  67. try {
  68. value[1][i] = cmd.transformReply(value[1][i]);
  69. }
  70. catch (err) {
  71. value[1][i] = err;
  72. }
  73. }
  74. }
  75. this._result[position] = value;
  76. if (--this.replyPending) {
  77. return;
  78. }
  79. if (this.isCluster) {
  80. let retriable = true;
  81. let commonError;
  82. for (let i = 0; i < this._result.length; ++i) {
  83. const error = this._result[i][0];
  84. const command = this._queue[i];
  85. if (error) {
  86. if (command.name === "exec" &&
  87. error.message ===
  88. "EXECABORT Transaction discarded because of previous errors.") {
  89. continue;
  90. }
  91. if (!commonError) {
  92. commonError = {
  93. name: error.name,
  94. message: error.message,
  95. };
  96. }
  97. else if (commonError.name !== error.name ||
  98. commonError.message !== error.message) {
  99. retriable = false;
  100. break;
  101. }
  102. }
  103. else if (!command.inTransaction) {
  104. const isReadOnly = (0, commands_1.exists)(command.name) && (0, commands_1.hasFlag)(command.name, "readonly");
  105. if (!isReadOnly) {
  106. retriable = false;
  107. break;
  108. }
  109. }
  110. }
  111. if (commonError && retriable) {
  112. const _this = this;
  113. const errv = commonError.message.split(" ");
  114. const queue = this._queue;
  115. let inTransaction = false;
  116. this._queue = [];
  117. for (let i = 0; i < queue.length; ++i) {
  118. if (errv[0] === "ASK" &&
  119. !inTransaction &&
  120. queue[i].name !== "asking" &&
  121. (!queue[i - 1] || queue[i - 1].name !== "asking")) {
  122. const asking = new Command_1.default("asking");
  123. asking.ignore = true;
  124. this.sendCommand(asking);
  125. }
  126. queue[i].initPromise();
  127. this.sendCommand(queue[i]);
  128. inTransaction = queue[i].inTransaction;
  129. }
  130. let matched = true;
  131. if (typeof this.leftRedirections === "undefined") {
  132. this.leftRedirections = {};
  133. }
  134. const exec = function () {
  135. _this.exec();
  136. };
  137. const cluster = this.redis;
  138. cluster.handleError(commonError, this.leftRedirections, {
  139. moved: function (_slot, key) {
  140. _this.preferKey = key;
  141. cluster.slots[errv[1]] = [key];
  142. cluster._groupsBySlot[errv[1]] =
  143. cluster._groupsIds[cluster.slots[errv[1]].join(";")];
  144. cluster.refreshSlotsCache();
  145. _this.exec();
  146. },
  147. ask: function (_slot, key) {
  148. _this.preferKey = key;
  149. _this.exec();
  150. },
  151. tryagain: exec,
  152. clusterDown: exec,
  153. connectionClosed: exec,
  154. maxRedirections: () => {
  155. matched = false;
  156. },
  157. defaults: () => {
  158. matched = false;
  159. },
  160. });
  161. if (matched) {
  162. return;
  163. }
  164. }
  165. }
  166. let ignoredCount = 0;
  167. for (let i = 0; i < this._queue.length - ignoredCount; ++i) {
  168. if (this._queue[i + ignoredCount].ignore) {
  169. ignoredCount += 1;
  170. }
  171. this._result[i] = this._result[i + ignoredCount];
  172. }
  173. this.resolve(this._result.slice(0, this._result.length - ignoredCount));
  174. }
  175. sendCommand(command) {
  176. if (this._transactions > 0) {
  177. command.inTransaction = true;
  178. }
  179. const position = this._queue.length;
  180. command.pipelineIndex = position;
  181. command.promise
  182. .then((result) => {
  183. this.fillResult([null, result], position);
  184. })
  185. .catch((error) => {
  186. this.fillResult([error], position);
  187. });
  188. this._queue.push(command);
  189. return this;
  190. }
  191. addBatch(commands) {
  192. let command, commandName, args;
  193. for (let i = 0; i < commands.length; ++i) {
  194. command = commands[i];
  195. commandName = command[0];
  196. args = command.slice(1);
  197. this[commandName].apply(this, args);
  198. }
  199. return this;
  200. }
  201. }
  202. exports.default = Pipeline;
  203. // @ts-expect-error
  204. const multi = Pipeline.prototype.multi;
  205. // @ts-expect-error
  206. Pipeline.prototype.multi = function () {
  207. this._transactions += 1;
  208. return multi.apply(this, arguments);
  209. };
  210. // @ts-expect-error
  211. const execBuffer = Pipeline.prototype.execBuffer;
  212. // @ts-expect-error
  213. Pipeline.prototype.execBuffer = (0, util_1.deprecate)(function () {
  214. if (this._transactions > 0) {
  215. this._transactions -= 1;
  216. }
  217. return execBuffer.apply(this, arguments);
  218. }, "Pipeline#execBuffer: Use Pipeline#exec instead");
  219. // NOTE: To avoid an unhandled promise rejection, this will unconditionally always return this.promise,
  220. // which always has the rejection handled by standard-as-callback
  221. // adding the provided rejection callback.
  222. //
  223. // If a different promise instance were returned, that promise would cause its own unhandled promise rejection
  224. // errors, even if that promise unconditionally resolved to **the resolved value of** this.promise.
  225. Pipeline.prototype.exec = function (callback) {
  226. // Wait for the cluster to be connected, since we need nodes information before continuing
  227. if (this.isCluster && !this.redis.slots.length) {
  228. if (this.redis.status === "wait")
  229. this.redis.connect().catch(utils_1.noop);
  230. if (callback && !this.nodeifiedPromise) {
  231. this.nodeifiedPromise = true;
  232. (0, standard_as_callback_1.default)(this.promise, callback);
  233. }
  234. this.redis.delayUntilReady((err) => {
  235. if (err) {
  236. this.reject(err);
  237. return;
  238. }
  239. this.exec(callback);
  240. });
  241. return this.promise;
  242. }
  243. if (this._transactions > 0) {
  244. this._transactions -= 1;
  245. return execBuffer.apply(this, arguments);
  246. }
  247. if (!this.nodeifiedPromise) {
  248. this.nodeifiedPromise = true;
  249. (0, standard_as_callback_1.default)(this.promise, callback);
  250. }
  251. if (!this._queue.length) {
  252. this.resolve([]);
  253. }
  254. let pipelineSlot;
  255. if (this.isCluster) {
  256. // List of the first key for each command
  257. const sampleKeys = [];
  258. for (let i = 0; i < this._queue.length; i++) {
  259. const keys = this._queue[i].getKeys();
  260. if (keys.length) {
  261. sampleKeys.push(keys[0]);
  262. }
  263. // For each command, check that the keys belong to the same slot
  264. if (keys.length && calculateSlot.generateMulti(keys) < 0) {
  265. this.reject(new Error("All the keys in a pipeline command should belong to the same slot"));
  266. return this.promise;
  267. }
  268. }
  269. if (sampleKeys.length) {
  270. pipelineSlot = generateMultiWithNodes(this.redis, sampleKeys);
  271. if (pipelineSlot < 0) {
  272. this.reject(new Error("All keys in the pipeline should belong to the same slots allocation group"));
  273. return this.promise;
  274. }
  275. }
  276. else {
  277. // Send the pipeline to a random node
  278. pipelineSlot = (Math.random() * 16384) | 0;
  279. }
  280. }
  281. const _this = this;
  282. execPipeline();
  283. return this.promise;
  284. function execPipeline() {
  285. let writePending = (_this.replyPending = _this._queue.length);
  286. let node;
  287. if (_this.isCluster) {
  288. node = {
  289. slot: pipelineSlot,
  290. redis: _this.redis.connectionPool.nodes.all[_this.preferKey],
  291. };
  292. }
  293. let data = "";
  294. let buffers;
  295. const stream = {
  296. isPipeline: true,
  297. destination: _this.isCluster ? node : { redis: _this.redis },
  298. write(writable) {
  299. if (typeof writable !== "string") {
  300. if (!buffers) {
  301. buffers = [];
  302. }
  303. if (data) {
  304. buffers.push(Buffer.from(data, "utf8"));
  305. data = "";
  306. }
  307. buffers.push(writable);
  308. }
  309. else {
  310. data += writable;
  311. }
  312. if (!--writePending) {
  313. if (buffers) {
  314. if (data) {
  315. buffers.push(Buffer.from(data, "utf8"));
  316. }
  317. stream.destination.redis.stream.write(Buffer.concat(buffers));
  318. }
  319. else {
  320. stream.destination.redis.stream.write(data);
  321. }
  322. // Reset writePending for resending
  323. writePending = _this._queue.length;
  324. data = "";
  325. buffers = undefined;
  326. }
  327. },
  328. };
  329. for (let i = 0; i < _this._queue.length; ++i) {
  330. _this.redis.sendCommand(_this._queue[i], stream, node);
  331. }
  332. return _this.promise;
  333. }
  334. };