Downloader.cpp 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. #include "Downloader.h"
  2. #include <curl/curl.h>
  3. #include <pthread.h>
  4. #include <sstream>
  5. #include "Bancho.h"
  6. #include "BanchoNetworking.h"
  7. #include "BanchoProtocol.h"
  8. #include "ConVar.h"
  9. #include "Engine.h"
  10. #include "miniz.h"
  11. struct DownloadResult {
  12. std::string url;
  13. std::vector<uint8_t> data;
  14. float progress = 0.f;
  15. int response_code = 0;
  16. };
  17. struct DownloadThread {
  18. bool running;
  19. pthread_t id;
  20. std::string endpoint;
  21. std::vector<DownloadResult*> downloads;
  22. };
  23. pthread_mutex_t threads_mtx = PTHREAD_MUTEX_INITIALIZER;
  24. std::vector<DownloadThread*> threads;
  25. void abort_downloads() {
  26. pthread_mutex_lock(&threads_mtx);
  27. for(auto thread : threads) {
  28. thread->running = false;
  29. }
  30. pthread_mutex_unlock(&threads_mtx);
  31. }
  32. void update_download_progress(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal,
  33. curl_off_t ulnow) {
  34. (void)ultotal;
  35. (void)ulnow;
  36. pthread_mutex_lock(&threads_mtx);
  37. auto result = (DownloadResult*)clientp;
  38. if(dltotal == 0) {
  39. result->progress = 0.f;
  40. } else if(dlnow > 0) {
  41. result->progress = (float)dlnow / (float)dltotal;
  42. }
  43. pthread_mutex_unlock(&threads_mtx);
  44. }
  45. void* do_downloads(void* arg) {
  46. auto thread = (DownloadThread*)arg;
  47. Packet response;
  48. CURL* curl = curl_easy_init();
  49. if(!curl) {
  50. debugLog("Failed to initialize cURL!\n");
  51. goto end_thread;
  52. }
  53. while(thread->running) {
  54. env->sleep(100000); // wait 100ms between every download
  55. DownloadResult* result = nullptr;
  56. std::string url;
  57. pthread_mutex_lock(&threads_mtx);
  58. for(auto download : thread->downloads) {
  59. if(download->progress == 0.f) {
  60. result = download;
  61. url = download->url;
  62. break;
  63. }
  64. }
  65. pthread_mutex_unlock(&threads_mtx);
  66. if(!result) continue;
  67. free(response.memory);
  68. response = Packet();
  69. debugLog("Downloading %s\n", url.c_str());
  70. curl_easy_reset(curl);
  71. curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
  72. curl_easy_setopt(curl, CURLOPT_USERAGENT, bancho.user_agent.toUtf8());
  73. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write);
  74. curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&response);
  75. curl_easy_setopt(curl, CURLOPT_XFERINFODATA, result);
  76. curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, update_download_progress);
  77. curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
  78. #ifdef _WIN32
  79. // ABSOLUTELY RETARDED, FUCK WINDOWS
  80. curl_easy_setopt(curl, CURLOPT_CAINFO, "curl-ca-bundle.crt");
  81. #endif
  82. CURLcode res = curl_easy_perform(curl);
  83. int response_code;
  84. curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
  85. if(res == CURLE_OK) {
  86. pthread_mutex_lock(&threads_mtx);
  87. result->progress = 1.f;
  88. result->response_code = response_code;
  89. result->data = std::vector<uint8_t>(response.memory, response.memory + response.size);
  90. pthread_mutex_unlock(&threads_mtx);
  91. } else {
  92. debugLog("Failed to download %s: %s\n", url.c_str(), curl_easy_strerror(res));
  93. pthread_mutex_lock(&threads_mtx);
  94. result->response_code = response_code;
  95. if(response_code == 429) {
  96. result->progress = 0.f;
  97. } else {
  98. result->data = std::vector<uint8_t>(response.memory, response.memory + response.size);
  99. result->progress = -1.f;
  100. }
  101. pthread_mutex_unlock(&threads_mtx);
  102. if(response_code == 429) {
  103. // Try again 5s later
  104. env->sleep(5000000);
  105. }
  106. }
  107. }
  108. end_thread:
  109. curl_easy_cleanup(curl);
  110. pthread_mutex_lock(&threads_mtx);
  111. std::vector<DownloadThread*> new_threads;
  112. for(auto dt : threads) {
  113. if(thread != dt) {
  114. new_threads.push_back(dt);
  115. }
  116. }
  117. threads = std::move(new_threads);
  118. pthread_mutex_unlock(&threads_mtx);
  119. free(response.memory);
  120. for(auto result : thread->downloads) {
  121. delete result;
  122. }
  123. delete thread;
  124. return NULL;
  125. }
  126. void download(const char* url, float* progress, std::vector<uint8_t>& out, int* response_code) {
  127. char* hostname = NULL;
  128. bool download_found = false;
  129. DownloadThread* matching_thread = nullptr;
  130. CURLU* urlu = curl_url();
  131. if(!urlu) {
  132. *progress = -1.f;
  133. return;
  134. }
  135. if(curl_url_set(urlu, CURLUPART_URL, url, 0) != CURLUE_OK) {
  136. *progress = -1.f;
  137. goto end;
  138. }
  139. if(curl_url_get(urlu, CURLUPART_HOST, &hostname, 0) != CURLUE_OK) {
  140. *progress = -1.f;
  141. goto end;
  142. }
  143. pthread_mutex_lock(&threads_mtx);
  144. for(auto thread : threads) {
  145. if(thread->running && !strcmp(thread->endpoint.c_str(), hostname)) {
  146. matching_thread = thread;
  147. break;
  148. }
  149. }
  150. if(matching_thread == nullptr) {
  151. matching_thread = new DownloadThread();
  152. matching_thread->running = true;
  153. matching_thread->endpoint = std::string(hostname);
  154. int ret = pthread_create(&matching_thread->id, NULL, do_downloads, matching_thread);
  155. if(ret) {
  156. debugLog("Failed to start download thread: pthread_create() returned %i\n", ret);
  157. *progress = -1.f;
  158. delete matching_thread;
  159. goto end;
  160. } else {
  161. threads.push_back(matching_thread);
  162. }
  163. }
  164. for(int i = 0; i < matching_thread->downloads.size(); i++) {
  165. DownloadResult* result = matching_thread->downloads[i];
  166. if(result->url == url) {
  167. *progress = result->progress;
  168. *response_code = result->response_code;
  169. if(*response_code != 0) {
  170. out = result->data;
  171. delete matching_thread->downloads[i];
  172. matching_thread->downloads.erase(matching_thread->downloads.begin() + i);
  173. }
  174. download_found = true;
  175. break;
  176. }
  177. }
  178. if(!download_found) {
  179. auto newdl = new DownloadResult{.url = url};
  180. matching_thread->downloads.push_back(newdl);
  181. *progress = 0.f;
  182. }
  183. pthread_mutex_unlock(&threads_mtx);
  184. end:
  185. curl_free(hostname);
  186. curl_url_cleanup(urlu);
  187. }
  188. void download_beatmapset(uint32_t set_id, float* progress) {
  189. // Check if we already have downloaded it
  190. std::stringstream ss;
  191. ss << MCENGINE_DATA_DIR "maps/" << std::to_string(set_id) << "/";
  192. auto map_dir = ss.str();
  193. if(env->directoryExists(map_dir)) {
  194. *progress = 1.f;
  195. return;
  196. }
  197. std::vector<uint8_t> data;
  198. auto mirror = convar->getConVarByName("beatmap_mirror")->getString();
  199. mirror.append(UString::format("%d", set_id));
  200. int response_code = 0;
  201. download(mirror.toUtf8(), progress, data, &response_code);
  202. if(response_code != 200) return;
  203. // Download succeeded: save map to disk
  204. mz_zip_archive zip = {0};
  205. mz_zip_archive_file_stat file_stat;
  206. mz_uint num_files = 0;
  207. debugLog("Extracting beatmapset %d (%d bytes)\n", set_id, data.size());
  208. if(!mz_zip_reader_init_mem(&zip, data.data(), data.size(), 0)) {
  209. debugLog("Failed to open .osz file\n");
  210. *progress = -1.f;
  211. return;
  212. }
  213. num_files = mz_zip_reader_get_num_files(&zip);
  214. if(num_files <= 0) {
  215. debugLog(".osz file is empty!\n");
  216. mz_zip_reader_end(&zip);
  217. *progress = -1.f;
  218. return;
  219. }
  220. if(!env->directoryExists(map_dir)) {
  221. env->createDirectory(map_dir);
  222. }
  223. for(mz_uint i = 0; i < num_files; i++) {
  224. if(!mz_zip_reader_file_stat(&zip, i, &file_stat)) continue;
  225. if(mz_zip_reader_is_file_a_directory(&zip, i)) continue;
  226. char* saveptr = NULL;
  227. char* folder = strtok_r(file_stat.m_filename, "/", &saveptr);
  228. std::string file_path = map_dir;
  229. while(folder != NULL) {
  230. if(!strcmp(folder, "..")) {
  231. // Bro...
  232. goto skip_file;
  233. }
  234. file_path.append("/");
  235. file_path.append(folder);
  236. folder = strtok_r(NULL, "/", &saveptr);
  237. if(folder != NULL) {
  238. if(!env->directoryExists(file_path)) {
  239. env->createDirectory(file_path);
  240. }
  241. }
  242. }
  243. mz_zip_reader_extract_to_file(&zip, i, file_path.c_str(), 0);
  244. skip_file:;
  245. // When a file can't be extracted we just ignore it (as long as the archive is valid).
  246. // We'll check for errors when loading the beatmap.
  247. }
  248. // Success
  249. mz_zip_reader_end(&zip);
  250. }