Downloader.cpp 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292
  1. #include "Downloader.h"
  2. #include <curl/curl.h>
  3. #include <pthread.h>
  4. #include <mutex>
  5. #include <sstream>
  6. #include "Bancho.h"
  7. #include "BanchoNetworking.h"
  8. #include "BanchoProtocol.h"
  9. #include "ConVar.h"
  10. #include "Engine.h"
  11. #include "miniz.h"
  12. struct DownloadResult {
  13. std::string url;
  14. std::vector<uint8_t> data;
  15. float progress = 0.f;
  16. int response_code = 0;
  17. };
  18. struct DownloadThread {
  19. bool running;
  20. pthread_t id;
  21. std::string endpoint;
  22. std::vector<DownloadResult*> downloads;
  23. };
  24. std::mutex threads_mtx;
  25. std::vector<DownloadThread*> threads;
  26. void abort_downloads() {
  27. threads_mtx.lock();
  28. for(auto thread : threads) {
  29. thread->running = false;
  30. }
  31. threads_mtx.unlock();
  32. }
  33. void update_download_progress(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal,
  34. curl_off_t ulnow) {
  35. (void)ultotal;
  36. (void)ulnow;
  37. threads_mtx.lock();
  38. auto result = (DownloadResult*)clientp;
  39. if(dltotal == 0) {
  40. result->progress = 0.f;
  41. } else if(dlnow > 0) {
  42. result->progress = (float)dlnow / (float)dltotal;
  43. }
  44. threads_mtx.unlock();
  45. }
  46. void* do_downloads(void* arg) {
  47. auto thread = (DownloadThread*)arg;
  48. Packet response;
  49. CURL* curl = curl_easy_init();
  50. if(!curl) {
  51. debugLog("Failed to initialize cURL!\n");
  52. goto end_thread;
  53. }
  54. while(thread->running) {
  55. env->sleep(100000); // wait 100ms between every download
  56. DownloadResult* result = nullptr;
  57. std::string url;
  58. threads_mtx.lock();
  59. for(auto download : thread->downloads) {
  60. if(download->progress == 0.f) {
  61. result = download;
  62. url = download->url;
  63. break;
  64. }
  65. }
  66. threads_mtx.unlock();
  67. if(!result) continue;
  68. free(response.memory);
  69. response = Packet();
  70. debugLog("Downloading %s\n", url.c_str());
  71. curl_easy_reset(curl);
  72. curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
  73. curl_easy_setopt(curl, CURLOPT_USERAGENT, bancho.user_agent.toUtf8());
  74. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write);
  75. curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&response);
  76. curl_easy_setopt(curl, CURLOPT_XFERINFODATA, result);
  77. curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, update_download_progress);
  78. curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
  79. #ifdef _WIN32
  80. // ABSOLUTELY RETARDED, FUCK WINDOWS
  81. curl_easy_setopt(curl, CURLOPT_CAINFO, "curl-ca-bundle.crt");
  82. #endif
  83. CURLcode res = curl_easy_perform(curl);
  84. int response_code;
  85. curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
  86. if(res == CURLE_OK) {
  87. threads_mtx.lock();
  88. result->progress = 1.f;
  89. result->response_code = response_code;
  90. result->data = std::vector<uint8_t>(response.memory, response.memory + response.size);
  91. threads_mtx.unlock();
  92. } else {
  93. debugLog("Failed to download %s: %s\n", url.c_str(), curl_easy_strerror(res));
  94. threads_mtx.lock();
  95. result->response_code = response_code;
  96. if(response_code == 429) {
  97. result->progress = 0.f;
  98. } else {
  99. result->data = std::vector<uint8_t>(response.memory, response.memory + response.size);
  100. result->progress = -1.f;
  101. }
  102. threads_mtx.unlock();
  103. if(response_code == 429) {
  104. // Try again 5s later
  105. env->sleep(5000000);
  106. }
  107. }
  108. }
  109. end_thread:
  110. curl_easy_cleanup(curl);
  111. threads_mtx.lock();
  112. std::vector<DownloadThread*> new_threads;
  113. for(auto dt : threads) {
  114. if(thread != dt) {
  115. new_threads.push_back(dt);
  116. }
  117. }
  118. threads = std::move(new_threads);
  119. threads_mtx.unlock();
  120. free(response.memory);
  121. for(auto result : thread->downloads) {
  122. delete result;
  123. }
  124. delete thread;
  125. return NULL;
  126. }
  127. void download(const char* url, float* progress, std::vector<uint8_t>& out, int* response_code) {
  128. char* hostname = NULL;
  129. bool download_found = false;
  130. DownloadThread* matching_thread = nullptr;
  131. CURLU* urlu = curl_url();
  132. if(!urlu) {
  133. *progress = -1.f;
  134. return;
  135. }
  136. if(curl_url_set(urlu, CURLUPART_URL, url, 0) != CURLUE_OK) {
  137. *progress = -1.f;
  138. goto end;
  139. }
  140. if(curl_url_get(urlu, CURLUPART_HOST, &hostname, 0) != CURLUE_OK) {
  141. *progress = -1.f;
  142. goto end;
  143. }
  144. threads_mtx.lock();
  145. for(auto thread : threads) {
  146. if(thread->running && !strcmp(thread->endpoint.c_str(), hostname)) {
  147. matching_thread = thread;
  148. break;
  149. }
  150. }
  151. if(matching_thread == nullptr) {
  152. matching_thread = new DownloadThread();
  153. matching_thread->running = true;
  154. matching_thread->endpoint = std::string(hostname);
  155. int ret = pthread_create(&matching_thread->id, NULL, do_downloads, matching_thread);
  156. if(ret) {
  157. debugLog("Failed to start download thread: pthread_create() returned %i\n", ret);
  158. *progress = -1.f;
  159. delete matching_thread;
  160. goto end;
  161. } else {
  162. threads.push_back(matching_thread);
  163. }
  164. }
  165. for(int i = 0; i < matching_thread->downloads.size(); i++) {
  166. DownloadResult* result = matching_thread->downloads[i];
  167. if(result->url == url) {
  168. *progress = result->progress;
  169. *response_code = result->response_code;
  170. if(result->progress == -1.f || result->progress == 1.f) {
  171. out = result->data;
  172. delete matching_thread->downloads[i];
  173. matching_thread->downloads.erase(matching_thread->downloads.begin() + i);
  174. }
  175. download_found = true;
  176. break;
  177. }
  178. }
  179. if(!download_found) {
  180. auto newdl = new DownloadResult{.url = url};
  181. matching_thread->downloads.push_back(newdl);
  182. *progress = 0.f;
  183. }
  184. threads_mtx.unlock();
  185. end:
  186. curl_url_cleanup(urlu);
  187. free(hostname);
  188. }
  189. void download_beatmapset(uint32_t set_id, float* progress) {
  190. // Check if we already have downloaded it
  191. std::stringstream ss;
  192. ss << MCENGINE_DATA_DIR "maps/" << std::to_string(set_id) << "/";
  193. auto map_dir = ss.str();
  194. if(env->directoryExists(map_dir)) {
  195. *progress = 1.f;
  196. return;
  197. }
  198. std::vector<uint8_t> data;
  199. auto mirror = convar->getConVarByName("beatmap_mirror")->getString();
  200. auto url = UString::format(mirror.toUtf8(), set_id);
  201. int response_code = 0;
  202. download(url.toUtf8(), progress, data, &response_code);
  203. if(response_code != 200) return;
  204. // Download succeeded: save map to disk
  205. mz_zip_archive zip = {0};
  206. mz_zip_archive_file_stat file_stat;
  207. mz_uint num_files = 0;
  208. debugLog("Extracting beatmapset %d (%d bytes)\n", set_id, data.size());
  209. if(!mz_zip_reader_init_mem(&zip, data.data(), data.size(), 0)) {
  210. debugLog("Failed to open .osz file\n");
  211. *progress = -1.f;
  212. return;
  213. }
  214. num_files = mz_zip_reader_get_num_files(&zip);
  215. if(num_files <= 0) {
  216. debugLog(".osz file is empty!\n");
  217. mz_zip_reader_end(&zip);
  218. *progress = -1.f;
  219. return;
  220. }
  221. if(!env->directoryExists(map_dir)) {
  222. env->createDirectory(map_dir);
  223. }
  224. for(mz_uint i = 0; i < num_files; i++) {
  225. if(!mz_zip_reader_file_stat(&zip, i, &file_stat)) continue;
  226. if(mz_zip_reader_is_file_a_directory(&zip, i)) continue;
  227. char* saveptr = NULL;
  228. char* folder = strtok_r(file_stat.m_filename, "/", &saveptr);
  229. std::string file_path = map_dir;
  230. while(folder != NULL) {
  231. if(!strcmp(folder, "..")) {
  232. // Bro...
  233. goto skip_file;
  234. }
  235. file_path.append("/");
  236. file_path.append(folder);
  237. folder = strtok_r(NULL, "/", &saveptr);
  238. if(folder != NULL) {
  239. if(!env->directoryExists(file_path)) {
  240. env->createDirectory(file_path);
  241. }
  242. }
  243. }
  244. mz_zip_reader_extract_to_file(&zip, i, file_path.c_str(), 0);
  245. skip_file:;
  246. // When a file can't be extracted we just ignore it (as long as the archive is valid).
  247. // We'll check for errors when loading the beatmap.
  248. }
  249. // Success
  250. mz_zip_reader_end(&zip);
  251. }