index.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396
  1. "use strict";
  2. var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
  3. function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
  4. return new (P || (P = Promise))(function (resolve, reject) {
  5. function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
  6. function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
  7. function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
  8. step((generator = generator.apply(thisArg, _arguments || [])).next());
  9. });
  10. };
  11. import { decode as base64Decode, encode as base64Encode } from "@ethersproject/base64";
  12. import { hexlify, isBytesLike } from "@ethersproject/bytes";
  13. import { shallowCopy } from "@ethersproject/properties";
  14. import { toUtf8Bytes, toUtf8String } from "@ethersproject/strings";
  15. import { Logger } from "@ethersproject/logger";
  16. import { version } from "./_version";
  17. const logger = new Logger(version);
  18. import { getUrl } from "./geturl";
  19. function staller(duration) {
  20. return new Promise((resolve) => {
  21. setTimeout(resolve, duration);
  22. });
  23. }
  24. function bodyify(value, type) {
  25. if (value == null) {
  26. return null;
  27. }
  28. if (typeof (value) === "string") {
  29. return value;
  30. }
  31. if (isBytesLike(value)) {
  32. if (type && (type.split("/")[0] === "text" || type.split(";")[0].trim() === "application/json")) {
  33. try {
  34. return toUtf8String(value);
  35. }
  36. catch (error) { }
  37. ;
  38. }
  39. return hexlify(value);
  40. }
  41. return value;
  42. }
  43. // This API is still a work in progress; the future changes will likely be:
  44. // - ConnectionInfo => FetchDataRequest<T = any>
  45. // - FetchDataRequest.body? = string | Uint8Array | { contentType: string, data: string | Uint8Array }
  46. // - If string => text/plain, Uint8Array => application/octet-stream (if content-type unspecified)
  47. // - FetchDataRequest.processFunc = (body: Uint8Array, response: FetchDataResponse) => T
  48. // For this reason, it should be considered internal until the API is finalized
  49. export function _fetchData(connection, body, processFunc) {
  50. // How many times to retry in the event of a throttle
  51. const attemptLimit = (typeof (connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit : 12;
  52. logger.assertArgument((attemptLimit > 0 && (attemptLimit % 1) === 0), "invalid connection throttle limit", "connection.throttleLimit", attemptLimit);
  53. const throttleCallback = ((typeof (connection) === "object") ? connection.throttleCallback : null);
  54. const throttleSlotInterval = ((typeof (connection) === "object" && typeof (connection.throttleSlotInterval) === "number") ? connection.throttleSlotInterval : 100);
  55. logger.assertArgument((throttleSlotInterval > 0 && (throttleSlotInterval % 1) === 0), "invalid connection throttle slot interval", "connection.throttleSlotInterval", throttleSlotInterval);
  56. const errorPassThrough = ((typeof (connection) === "object") ? !!(connection.errorPassThrough) : false);
  57. const headers = {};
  58. let url = null;
  59. // @TODO: Allow ConnectionInfo to override some of these values
  60. const options = {
  61. method: "GET",
  62. };
  63. let allow304 = false;
  64. let timeout = 2 * 60 * 1000;
  65. if (typeof (connection) === "string") {
  66. url = connection;
  67. }
  68. else if (typeof (connection) === "object") {
  69. if (connection == null || connection.url == null) {
  70. logger.throwArgumentError("missing URL", "connection.url", connection);
  71. }
  72. url = connection.url;
  73. if (typeof (connection.timeout) === "number" && connection.timeout > 0) {
  74. timeout = connection.timeout;
  75. }
  76. if (connection.headers) {
  77. for (const key in connection.headers) {
  78. headers[key.toLowerCase()] = { key: key, value: String(connection.headers[key]) };
  79. if (["if-none-match", "if-modified-since"].indexOf(key.toLowerCase()) >= 0) {
  80. allow304 = true;
  81. }
  82. }
  83. }
  84. options.allowGzip = !!connection.allowGzip;
  85. if (connection.user != null && connection.password != null) {
  86. if (url.substring(0, 6) !== "https:" && connection.allowInsecureAuthentication !== true) {
  87. logger.throwError("basic authentication requires a secure https url", Logger.errors.INVALID_ARGUMENT, { argument: "url", url: url, user: connection.user, password: "[REDACTED]" });
  88. }
  89. const authorization = connection.user + ":" + connection.password;
  90. headers["authorization"] = {
  91. key: "Authorization",
  92. value: "Basic " + base64Encode(toUtf8Bytes(authorization))
  93. };
  94. }
  95. if (connection.skipFetchSetup != null) {
  96. options.skipFetchSetup = !!connection.skipFetchSetup;
  97. }
  98. }
  99. const reData = new RegExp("^data:([a-z0-9-]+/[a-z0-9-]+);base64,(.*)$", "i");
  100. const dataMatch = ((url) ? url.match(reData) : null);
  101. if (dataMatch) {
  102. try {
  103. const response = {
  104. statusCode: 200,
  105. statusMessage: "OK",
  106. headers: { "content-type": dataMatch[1] },
  107. body: base64Decode(dataMatch[2])
  108. };
  109. let result = response.body;
  110. if (processFunc) {
  111. result = processFunc(response.body, response);
  112. }
  113. return Promise.resolve(result);
  114. }
  115. catch (error) {
  116. logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
  117. body: bodyify(dataMatch[1], dataMatch[2]),
  118. error: error,
  119. requestBody: null,
  120. requestMethod: "GET",
  121. url: url
  122. });
  123. }
  124. }
  125. if (body) {
  126. options.method = "POST";
  127. options.body = body;
  128. if (headers["content-type"] == null) {
  129. headers["content-type"] = { key: "Content-Type", value: "application/octet-stream" };
  130. }
  131. if (headers["content-length"] == null) {
  132. headers["content-length"] = { key: "Content-Length", value: String(body.length) };
  133. }
  134. }
  135. const flatHeaders = {};
  136. Object.keys(headers).forEach((key) => {
  137. const header = headers[key];
  138. flatHeaders[header.key] = header.value;
  139. });
  140. options.headers = flatHeaders;
  141. const runningTimeout = (function () {
  142. let timer = null;
  143. const promise = new Promise(function (resolve, reject) {
  144. if (timeout) {
  145. timer = setTimeout(() => {
  146. if (timer == null) {
  147. return;
  148. }
  149. timer = null;
  150. reject(logger.makeError("timeout", Logger.errors.TIMEOUT, {
  151. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  152. requestMethod: options.method,
  153. timeout: timeout,
  154. url: url
  155. }));
  156. }, timeout);
  157. }
  158. });
  159. const cancel = function () {
  160. if (timer == null) {
  161. return;
  162. }
  163. clearTimeout(timer);
  164. timer = null;
  165. };
  166. return { promise, cancel };
  167. })();
  168. const runningFetch = (function () {
  169. return __awaiter(this, void 0, void 0, function* () {
  170. for (let attempt = 0; attempt < attemptLimit; attempt++) {
  171. let response = null;
  172. try {
  173. response = yield getUrl(url, options);
  174. if (attempt < attemptLimit) {
  175. if (response.statusCode === 301 || response.statusCode === 302) {
  176. // Redirection; for now we only support absolute locataions
  177. const location = response.headers.location || "";
  178. if (options.method === "GET" && location.match(/^https:/)) {
  179. url = response.headers.location;
  180. continue;
  181. }
  182. }
  183. else if (response.statusCode === 429) {
  184. // Exponential back-off throttling
  185. let tryAgain = true;
  186. if (throttleCallback) {
  187. tryAgain = yield throttleCallback(attempt, url);
  188. }
  189. if (tryAgain) {
  190. let stall = 0;
  191. const retryAfter = response.headers["retry-after"];
  192. if (typeof (retryAfter) === "string" && retryAfter.match(/^[1-9][0-9]*$/)) {
  193. stall = parseInt(retryAfter) * 1000;
  194. }
  195. else {
  196. stall = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  197. }
  198. //console.log("Stalling 429");
  199. yield staller(stall);
  200. continue;
  201. }
  202. }
  203. }
  204. }
  205. catch (error) {
  206. response = error.response;
  207. if (response == null) {
  208. runningTimeout.cancel();
  209. logger.throwError("missing response", Logger.errors.SERVER_ERROR, {
  210. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  211. requestMethod: options.method,
  212. serverError: error,
  213. url: url
  214. });
  215. }
  216. }
  217. let body = response.body;
  218. if (allow304 && response.statusCode === 304) {
  219. body = null;
  220. }
  221. else if (!errorPassThrough && (response.statusCode < 200 || response.statusCode >= 300)) {
  222. runningTimeout.cancel();
  223. logger.throwError("bad response", Logger.errors.SERVER_ERROR, {
  224. status: response.statusCode,
  225. headers: response.headers,
  226. body: bodyify(body, ((response.headers) ? response.headers["content-type"] : null)),
  227. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  228. requestMethod: options.method,
  229. url: url
  230. });
  231. }
  232. if (processFunc) {
  233. try {
  234. const result = yield processFunc(body, response);
  235. runningTimeout.cancel();
  236. return result;
  237. }
  238. catch (error) {
  239. // Allow the processFunc to trigger a throttle
  240. if (error.throttleRetry && attempt < attemptLimit) {
  241. let tryAgain = true;
  242. if (throttleCallback) {
  243. tryAgain = yield throttleCallback(attempt, url);
  244. }
  245. if (tryAgain) {
  246. const timeout = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  247. //console.log("Stalling callback");
  248. yield staller(timeout);
  249. continue;
  250. }
  251. }
  252. runningTimeout.cancel();
  253. logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
  254. body: bodyify(body, ((response.headers) ? response.headers["content-type"] : null)),
  255. error: error,
  256. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  257. requestMethod: options.method,
  258. url: url
  259. });
  260. }
  261. }
  262. runningTimeout.cancel();
  263. // If we had a processFunc, it either returned a T or threw above.
  264. // The "body" is now a Uint8Array.
  265. return body;
  266. }
  267. return logger.throwError("failed response", Logger.errors.SERVER_ERROR, {
  268. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  269. requestMethod: options.method,
  270. url: url
  271. });
  272. });
  273. })();
  274. return Promise.race([runningTimeout.promise, runningFetch]);
  275. }
  276. export function fetchJson(connection, json, processFunc) {
  277. let processJsonFunc = (value, response) => {
  278. let result = null;
  279. if (value != null) {
  280. try {
  281. result = JSON.parse(toUtf8String(value));
  282. }
  283. catch (error) {
  284. logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
  285. body: value,
  286. error: error
  287. });
  288. }
  289. }
  290. if (processFunc) {
  291. result = processFunc(result, response);
  292. }
  293. return result;
  294. };
  295. // If we have json to send, we must
  296. // - add content-type of application/json (unless already overridden)
  297. // - convert the json to bytes
  298. let body = null;
  299. if (json != null) {
  300. body = toUtf8Bytes(json);
  301. // Create a connection with the content-type set for JSON
  302. const updated = (typeof (connection) === "string") ? ({ url: connection }) : shallowCopy(connection);
  303. if (updated.headers) {
  304. const hasContentType = (Object.keys(updated.headers).filter((k) => (k.toLowerCase() === "content-type")).length) !== 0;
  305. if (!hasContentType) {
  306. updated.headers = shallowCopy(updated.headers);
  307. updated.headers["content-type"] = "application/json";
  308. }
  309. }
  310. else {
  311. updated.headers = { "content-type": "application/json" };
  312. }
  313. connection = updated;
  314. }
  315. return _fetchData(connection, body, processJsonFunc);
  316. }
  317. export function poll(func, options) {
  318. if (!options) {
  319. options = {};
  320. }
  321. options = shallowCopy(options);
  322. if (options.floor == null) {
  323. options.floor = 0;
  324. }
  325. if (options.ceiling == null) {
  326. options.ceiling = 10000;
  327. }
  328. if (options.interval == null) {
  329. options.interval = 250;
  330. }
  331. return new Promise(function (resolve, reject) {
  332. let timer = null;
  333. let done = false;
  334. // Returns true if cancel was successful. Unsuccessful cancel means we're already done.
  335. const cancel = () => {
  336. if (done) {
  337. return false;
  338. }
  339. done = true;
  340. if (timer) {
  341. clearTimeout(timer);
  342. }
  343. return true;
  344. };
  345. if (options.timeout) {
  346. timer = setTimeout(() => {
  347. if (cancel()) {
  348. reject(new Error("timeout"));
  349. }
  350. }, options.timeout);
  351. }
  352. const retryLimit = options.retryLimit;
  353. let attempt = 0;
  354. function check() {
  355. return func().then(function (result) {
  356. // If we have a result, or are allowed null then we're done
  357. if (result !== undefined) {
  358. if (cancel()) {
  359. resolve(result);
  360. }
  361. }
  362. else if (options.oncePoll) {
  363. options.oncePoll.once("poll", check);
  364. }
  365. else if (options.onceBlock) {
  366. options.onceBlock.once("block", check);
  367. // Otherwise, exponential back-off (up to 10s) our next request
  368. }
  369. else if (!done) {
  370. attempt++;
  371. if (attempt > retryLimit) {
  372. if (cancel()) {
  373. reject(new Error("retry limit reached"));
  374. }
  375. return;
  376. }
  377. let timeout = options.interval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  378. if (timeout < options.floor) {
  379. timeout = options.floor;
  380. }
  381. if (timeout > options.ceiling) {
  382. timeout = options.ceiling;
  383. }
  384. setTimeout(check, timeout);
  385. }
  386. return null;
  387. }, function (error) {
  388. if (cancel()) {
  389. reject(error);
  390. }
  391. });
  392. }
  393. check();
  394. });
  395. }
  396. //# sourceMappingURL=index.js.map