| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1 | #include "cache.h" |
| 2 | #include "commit.h" |
| Junio C Hamano | 271421c | 2005-09-30 07:07:39 | [diff] [blame] | 3 | #include "pack.h" |
| Junio C Hamano | 215a7ad | 2005-09-08 00:26:23 | [diff] [blame] | 4 | #include "fetch.h" |
| Daniel Barkalow | 4250a5e | 2005-04-30 23:53:56 | [diff] [blame] | 5 | |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 6 | #include <curl/curl.h> |
| 7 | #include <curl/easy.h> |
| 8 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 9 | #if LIBCURL_VERSION_NUM >= 0x070908 |
| 10 | #define USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 11 | #define DEFAULT_MAX_REQUESTS 5 |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 12 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 13 | |
| Johannes Schindelin | 4a30976 | 2005-07-29 02:24:13 | [diff] [blame] | 14 | #if LIBCURL_VERSION_NUM < 0x070704 |
| 15 | #define curl_global_cleanup() do { /* nothing */ } while(0) |
| 16 | #endif |
| 17 | #if LIBCURL_VERSION_NUM < 0x070800 |
| 18 | #define curl_global_init(a) do { /* nothing */ } while(0) |
| 19 | #endif |
| Johannes Schindelin | 4a30976 | 2005-07-29 02:24:13 | [diff] [blame] | 20 | |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 21 | #if LIBCURL_VERSION_NUM < 0x070c04 |
| 22 | #define NO_CURL_EASY_DUPHANDLE |
| 23 | #endif |
| 24 | |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 25 | #define PREV_BUF_SIZE 4096 |
| 26 | #define RANGE_HEADER_SIZE 30 |
| 27 | |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 28 | static int got_alternates = 0; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 29 | static int active_requests = 0; |
| 30 | static int data_received; |
| 31 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 32 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 33 | static int max_requests = -1; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 34 | static CURLM *curlm; |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 35 | #endif |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 36 | #ifndef NO_CURL_EASY_DUPHANDLE |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 37 | static CURL *curl_default; |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 38 | #endif |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 39 | static struct curl_slist *pragma_header; |
| Sergey Vlasov | 1db69b5 | 2005-09-13 15:38:58 | [diff] [blame] | 40 | static struct curl_slist *no_pragma_header; |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 41 | static struct curl_slist *no_range_header; |
| Nick Hengeveld | 1ddea77 | 2005-09-26 17:52:11 | [diff] [blame] | 42 | static char curl_errorstr[CURL_ERROR_SIZE]; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 43 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 44 | struct alt_base |
| 45 | { |
| 46 | char *base; |
| 47 | int got_indices; |
| 48 | struct packed_git *packs; |
| 49 | struct alt_base *next; |
| 50 | }; |
| 51 | |
| Peter Hagervall | a7928f8 | 2005-09-28 12:04:54 | [diff] [blame] | 52 | static struct alt_base *alt = NULL; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 53 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 54 | enum transfer_state { |
| 55 | WAITING, |
| 56 | ABORTED, |
| 57 | ACTIVE, |
| 58 | COMPLETE, |
| 59 | }; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 60 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 61 | struct transfer_request |
| 62 | { |
| 63 | unsigned char sha1[20]; |
| 64 | struct alt_base *repo; |
| 65 | char *url; |
| 66 | char filename[PATH_MAX]; |
| 67 | char tmpfile[PATH_MAX]; |
| 68 | int local; |
| 69 | enum transfer_state state; |
| 70 | CURLcode curl_result; |
| 71 | char errorstr[CURL_ERROR_SIZE]; |
| 72 | long http_code; |
| 73 | unsigned char real_sha1[20]; |
| 74 | SHA_CTX c; |
| 75 | z_stream stream; |
| 76 | int zret; |
| 77 | int rename; |
| 78 | struct active_request_slot *slot; |
| 79 | struct transfer_request *next; |
| 80 | }; |
| 81 | |
| 82 | struct active_request_slot |
| 83 | { |
| 84 | CURL *curl; |
| 85 | FILE *local; |
| 86 | int in_use; |
| 87 | int done; |
| 88 | CURLcode curl_result; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 89 | long http_code; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 90 | struct active_request_slot *next; |
| 91 | }; |
| 92 | |
| 93 | static struct transfer_request *request_queue_head = NULL; |
| 94 | static struct active_request_slot *active_queue_head = NULL; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 95 | |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 96 | static int curl_ssl_verify = -1; |
| 97 | static char *ssl_cert = NULL; |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 98 | #if LIBCURL_VERSION_NUM >= 0x070902 |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 99 | static char *ssl_key = NULL; |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 100 | #endif |
| 101 | #if LIBCURL_VERSION_NUM >= 0x070908 |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 102 | static char *ssl_capath = NULL; |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 103 | #endif |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 104 | static char *ssl_cainfo = NULL; |
| Nick Hengeveld | e0004e2 | 2005-10-19 21:27:01 | [diff] [blame] | 105 | static long curl_low_speed_limit = -1; |
| 106 | static long curl_low_speed_time = -1; |
| Darrin Thompson | 3dcb90f | 2005-07-13 02:12:40 | [diff] [blame] | 107 | |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 108 | struct buffer |
| 109 | { |
| 110 | size_t posn; |
| 111 | size_t size; |
| 112 | void *buffer; |
| 113 | }; |
| 114 | |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 115 | static int http_options(const char *var, const char *value) |
| 116 | { |
| 117 | if (!strcmp("http.sslverify", var)) { |
| 118 | if (curl_ssl_verify == -1) { |
| 119 | curl_ssl_verify = git_config_bool(var, value); |
| 120 | } |
| 121 | return 0; |
| 122 | } |
| 123 | |
| 124 | if (!strcmp("http.sslcert", var)) { |
| 125 | if (ssl_cert == NULL) { |
| 126 | ssl_cert = xmalloc(strlen(value)+1); |
| 127 | strcpy(ssl_cert, value); |
| 128 | } |
| 129 | return 0; |
| 130 | } |
| 131 | #if LIBCURL_VERSION_NUM >= 0x070902 |
| 132 | if (!strcmp("http.sslkey", var)) { |
| 133 | if (ssl_key == NULL) { |
| 134 | ssl_key = xmalloc(strlen(value)+1); |
| 135 | strcpy(ssl_key, value); |
| 136 | } |
| 137 | return 0; |
| 138 | } |
| 139 | #endif |
| 140 | #if LIBCURL_VERSION_NUM >= 0x070908 |
| 141 | if (!strcmp("http.sslcapath", var)) { |
| 142 | if (ssl_capath == NULL) { |
| 143 | ssl_capath = xmalloc(strlen(value)+1); |
| 144 | strcpy(ssl_capath, value); |
| 145 | } |
| 146 | return 0; |
| 147 | } |
| 148 | #endif |
| 149 | if (!strcmp("http.sslcainfo", var)) { |
| 150 | if (ssl_cainfo == NULL) { |
| 151 | ssl_cainfo = xmalloc(strlen(value)+1); |
| 152 | strcpy(ssl_cainfo, value); |
| 153 | } |
| 154 | return 0; |
| 155 | } |
| 156 | |
| 157 | #ifdef USE_CURL_MULTI |
| 158 | if (!strcmp("http.maxrequests", var)) { |
| 159 | if (max_requests == -1) |
| 160 | max_requests = git_config_int(var, value); |
| 161 | return 0; |
| 162 | } |
| 163 | #endif |
| 164 | |
| Nick Hengeveld | e0004e2 | 2005-10-19 21:27:01 | [diff] [blame] | 165 | if (!strcmp("http.lowspeedlimit", var)) { |
| 166 | if (curl_low_speed_limit == -1) |
| 167 | curl_low_speed_limit = (long)git_config_int(var, value); |
| 168 | return 0; |
| 169 | } |
| 170 | if (!strcmp("http.lowspeedtime", var)) { |
| 171 | if (curl_low_speed_time == -1) |
| 172 | curl_low_speed_time = (long)git_config_int(var, value); |
| 173 | return 0; |
| 174 | } |
| 175 | |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 176 | /* Fall back on the default ones */ |
| 177 | return git_default_config(var, value); |
| 178 | } |
| 179 | |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 180 | static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 181 | struct buffer *buffer) |
| 182 | { |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 183 | size_t size = eltsize * nmemb; |
| 184 | if (size > buffer->size - buffer->posn) |
| 185 | size = buffer->size - buffer->posn; |
| 186 | memcpy(buffer->buffer + buffer->posn, ptr, size); |
| 187 | buffer->posn += size; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 188 | data_received++; |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 189 | return size; |
| 190 | } |
| 191 | |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 192 | static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize, |
| 193 | size_t nmemb, struct buffer *buffer) |
| 194 | { |
| 195 | size_t size = eltsize * nmemb; |
| 196 | if (size > buffer->size - buffer->posn) { |
| 197 | buffer->size = buffer->size * 3 / 2; |
| 198 | if (buffer->size < buffer->posn + size) |
| 199 | buffer->size = buffer->posn + size; |
| 200 | buffer->buffer = xrealloc(buffer->buffer, buffer->size); |
| 201 | } |
| 202 | memcpy(buffer->buffer + buffer->posn, ptr, size); |
| 203 | buffer->posn += size; |
| 204 | data_received++; |
| 205 | return size; |
| 206 | } |
| 207 | |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 208 | static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, |
| 209 | void *data) |
| 210 | { |
| Brian Gerst | bf0f910 | 2005-05-18 12:14:09 | [diff] [blame] | 211 | unsigned char expn[4096]; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 212 | size_t size = eltsize * nmemb; |
| 213 | int posn = 0; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 214 | struct transfer_request *request = (struct transfer_request *)data; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 215 | do { |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 216 | ssize_t retval = write(request->local, |
| 217 | ptr + posn, size - posn); |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 218 | if (retval < 0) |
| 219 | return posn; |
| 220 | posn += retval; |
| 221 | } while (posn < size); |
| 222 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 223 | request->stream.avail_in = size; |
| 224 | request->stream.next_in = ptr; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 225 | do { |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 226 | request->stream.next_out = expn; |
| 227 | request->stream.avail_out = sizeof(expn); |
| 228 | request->zret = inflate(&request->stream, Z_SYNC_FLUSH); |
| 229 | SHA1_Update(&request->c, expn, |
| 230 | sizeof(expn) - request->stream.avail_out); |
| 231 | } while (request->stream.avail_in && request->zret == Z_OK); |
| 232 | data_received++; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 233 | return size; |
| 234 | } |
| 235 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 236 | #ifdef USE_CURL_MULTI |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 237 | static void process_curl_messages(void); |
| 238 | static void process_request_queue(void); |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 239 | #endif |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 240 | static int fetch_alternates(char *base); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 241 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 242 | static CURL* get_curl_handle(void) |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 243 | { |
| 244 | CURL* result = curl_easy_init(); |
| 245 | |
| 246 | curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); |
| 247 | #if LIBCURL_VERSION_NUM >= 0x070907 |
| 248 | curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); |
| 249 | #endif |
| 250 | |
| 251 | if (ssl_cert != NULL) |
| 252 | curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); |
| 253 | #if LIBCURL_VERSION_NUM >= 0x070902 |
| 254 | if (ssl_key != NULL) |
| 255 | curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); |
| 256 | #endif |
| 257 | #if LIBCURL_VERSION_NUM >= 0x070908 |
| 258 | if (ssl_capath != NULL) |
| 259 | curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); |
| 260 | #endif |
| 261 | if (ssl_cainfo != NULL) |
| 262 | curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); |
| 263 | curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); |
| 264 | |
| Nick Hengeveld | e0004e2 | 2005-10-19 21:27:01 | [diff] [blame] | 265 | if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { |
| 266 | curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, |
| 267 | curl_low_speed_limit); |
| 268 | curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, |
| 269 | curl_low_speed_time); |
| 270 | } |
| 271 | |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 272 | return result; |
| 273 | } |
| 274 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 275 | static struct active_request_slot *get_active_slot(void) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 276 | { |
| 277 | struct active_request_slot *slot = active_queue_head; |
| 278 | struct active_request_slot *newslot; |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 279 | |
| 280 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 281 | int num_transfers; |
| 282 | |
| 283 | /* Wait for a slot to open up if the queue is full */ |
| 284 | while (active_requests >= max_requests) { |
| 285 | curl_multi_perform(curlm, &num_transfers); |
| 286 | if (num_transfers < active_requests) { |
| 287 | process_curl_messages(); |
| 288 | } |
| 289 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 290 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 291 | |
| 292 | while (slot != NULL && slot->in_use) { |
| 293 | slot = slot->next; |
| 294 | } |
| 295 | if (slot == NULL) { |
| 296 | newslot = xmalloc(sizeof(*newslot)); |
| Nick Hengeveld | f1a906a | 2005-10-21 19:06:10 | [diff] [blame] | 297 | newslot->curl = NULL; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 298 | newslot->in_use = 0; |
| 299 | newslot->next = NULL; |
| 300 | |
| 301 | slot = active_queue_head; |
| 302 | if (slot == NULL) { |
| 303 | active_queue_head = newslot; |
| 304 | } else { |
| 305 | while (slot->next != NULL) { |
| 306 | slot = slot->next; |
| 307 | } |
| 308 | slot->next = newslot; |
| 309 | } |
| 310 | slot = newslot; |
| 311 | } |
| 312 | |
| Nick Hengeveld | f1a906a | 2005-10-21 19:06:10 | [diff] [blame] | 313 | if (slot->curl == NULL) { |
| 314 | #ifdef NO_CURL_EASY_DUPHANDLE |
| 315 | slot->curl = get_curl_handle(); |
| 316 | #else |
| 317 | slot->curl = curl_easy_duphandle(curl_default); |
| 318 | #endif |
| 319 | } |
| 320 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 321 | active_requests++; |
| 322 | slot->in_use = 1; |
| 323 | slot->done = 0; |
| 324 | slot->local = NULL; |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 325 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 326 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); |
| 327 | curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); |
| 328 | |
| 329 | return slot; |
| 330 | } |
| 331 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 332 | static int start_active_slot(struct active_request_slot *slot) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 333 | { |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 334 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 335 | CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); |
| 336 | |
| 337 | if (curlm_result != CURLM_OK && |
| 338 | curlm_result != CURLM_CALL_MULTI_PERFORM) { |
| 339 | active_requests--; |
| 340 | slot->in_use = 0; |
| 341 | return 0; |
| 342 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 343 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 344 | return 1; |
| 345 | } |
| 346 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 347 | static void run_active_slot(struct active_request_slot *slot) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 348 | { |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 349 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 350 | int num_transfers; |
| 351 | long last_pos = 0; |
| 352 | long current_pos; |
| 353 | fd_set readfds; |
| 354 | fd_set writefds; |
| 355 | fd_set excfds; |
| 356 | int max_fd; |
| 357 | struct timeval select_timeout; |
| 358 | CURLMcode curlm_result; |
| 359 | |
| 360 | while (!slot->done) { |
| 361 | data_received = 0; |
| 362 | do { |
| 363 | curlm_result = curl_multi_perform(curlm, |
| 364 | &num_transfers); |
| 365 | } while (curlm_result == CURLM_CALL_MULTI_PERFORM); |
| 366 | if (num_transfers < active_requests) { |
| 367 | process_curl_messages(); |
| 368 | process_request_queue(); |
| 369 | } |
| 370 | |
| 371 | if (!data_received && slot->local != NULL) { |
| 372 | current_pos = ftell(slot->local); |
| 373 | if (current_pos > last_pos) |
| 374 | data_received++; |
| 375 | last_pos = current_pos; |
| 376 | } |
| 377 | |
| 378 | if (!slot->done && !data_received) { |
| 379 | max_fd = 0; |
| 380 | FD_ZERO(&readfds); |
| 381 | FD_ZERO(&writefds); |
| 382 | FD_ZERO(&excfds); |
| 383 | select_timeout.tv_sec = 0; |
| 384 | select_timeout.tv_usec = 50000; |
| 385 | select(max_fd, &readfds, &writefds, |
| 386 | &excfds, &select_timeout); |
| 387 | } |
| 388 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 389 | #else |
| 390 | slot->curl_result = curl_easy_perform(slot->curl); |
| 391 | active_requests--; |
| 392 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 393 | } |
| 394 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 395 | static void start_request(struct transfer_request *request) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 396 | { |
| 397 | char *hex = sha1_to_hex(request->sha1); |
| 398 | char prevfile[PATH_MAX]; |
| 399 | char *url; |
| 400 | char *posn; |
| 401 | int prevlocal; |
| 402 | unsigned char prev_buf[PREV_BUF_SIZE]; |
| 403 | ssize_t prev_read = 0; |
| 404 | long prev_posn = 0; |
| 405 | char range[RANGE_HEADER_SIZE]; |
| 406 | struct curl_slist *range_header = NULL; |
| 407 | struct active_request_slot *slot; |
| 408 | |
| 409 | snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename); |
| 410 | unlink(prevfile); |
| 411 | rename(request->tmpfile, prevfile); |
| 412 | unlink(request->tmpfile); |
| 413 | |
| 414 | request->local = open(request->tmpfile, |
| 415 | O_WRONLY | O_CREAT | O_EXCL, 0666); |
| Junio C Hamano | b721e01 | 2005-10-11 06:22:01 | [diff] [blame] | 416 | /* This could have failed due to the "lazy directory creation"; |
| 417 | * try to mkdir the last path component. |
| 418 | */ |
| 419 | if (request->local < 0 && errno == ENOENT) { |
| 420 | char *dir = strrchr(request->tmpfile, '/'); |
| 421 | if (dir) { |
| 422 | *dir = 0; |
| 423 | mkdir(request->tmpfile, 0777); |
| 424 | *dir = '/'; |
| 425 | } |
| 426 | request->local = open(request->tmpfile, |
| 427 | O_WRONLY | O_CREAT | O_EXCL, 0666); |
| 428 | } |
| 429 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 430 | if (request->local < 0) { |
| 431 | request->state = ABORTED; |
| 432 | error("Couldn't create temporary file %s for %s: %s\n", |
| 433 | request->tmpfile, request->filename, strerror(errno)); |
| 434 | return; |
| 435 | } |
| 436 | |
| 437 | memset(&request->stream, 0, sizeof(request->stream)); |
| 438 | |
| 439 | inflateInit(&request->stream); |
| 440 | |
| 441 | SHA1_Init(&request->c); |
| 442 | |
| 443 | url = xmalloc(strlen(request->repo->base) + 50); |
| 444 | request->url = xmalloc(strlen(request->repo->base) + 50); |
| 445 | strcpy(url, request->repo->base); |
| 446 | posn = url + strlen(request->repo->base); |
| 447 | strcpy(posn, "objects/"); |
| 448 | posn += 8; |
| 449 | memcpy(posn, hex, 2); |
| 450 | posn += 2; |
| 451 | *(posn++) = '/'; |
| 452 | strcpy(posn, hex + 2); |
| 453 | strcpy(request->url, url); |
| 454 | |
| 455 | /* If a previous temp file is present, process what was already |
| 456 | fetched. */ |
| 457 | prevlocal = open(prevfile, O_RDONLY); |
| 458 | if (prevlocal != -1) { |
| 459 | do { |
| 460 | prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE); |
| 461 | if (prev_read>0) { |
| 462 | if (fwrite_sha1_file(prev_buf, |
| 463 | 1, |
| 464 | prev_read, |
| 465 | request) == prev_read) { |
| 466 | prev_posn += prev_read; |
| 467 | } else { |
| 468 | prev_read = -1; |
| 469 | } |
| 470 | } |
| 471 | } while (prev_read > 0); |
| 472 | close(prevlocal); |
| 473 | } |
| 474 | unlink(prevfile); |
| 475 | |
| 476 | /* Reset inflate/SHA1 if there was an error reading the previous temp |
| 477 | file; also rewind to the beginning of the local file. */ |
| 478 | if (prev_read == -1) { |
| 479 | memset(&request->stream, 0, sizeof(request->stream)); |
| 480 | inflateInit(&request->stream); |
| 481 | SHA1_Init(&request->c); |
| 482 | if (prev_posn>0) { |
| 483 | prev_posn = 0; |
| 484 | lseek(request->local, SEEK_SET, 0); |
| 485 | ftruncate(request->local, 0); |
| 486 | } |
| 487 | } |
| 488 | |
| 489 | slot = get_active_slot(); |
| 490 | curl_easy_setopt(slot->curl, CURLOPT_FILE, request); |
| 491 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); |
| 492 | curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr); |
| 493 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 494 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 495 | |
| 496 | /* If we have successfully processed data from a previous fetch |
| 497 | attempt, only fetch the data we don't already have. */ |
| 498 | if (prev_posn>0) { |
| 499 | if (get_verbosely) |
| 500 | fprintf(stderr, |
| 501 | "Resuming fetch of object %s at byte %ld\n", |
| 502 | hex, prev_posn); |
| 503 | sprintf(range, "Range: bytes=%ld-", prev_posn); |
| 504 | range_header = curl_slist_append(range_header, range); |
| 505 | curl_easy_setopt(slot->curl, |
| 506 | CURLOPT_HTTPHEADER, range_header); |
| 507 | } |
| 508 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 509 | /* Try to get the request started, abort the request on error */ |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 510 | if (!start_active_slot(slot)) { |
| 511 | request->state = ABORTED; |
| 512 | close(request->local); |
| 513 | free(request->url); |
| 514 | return; |
| 515 | } |
| 516 | |
| 517 | request->slot = slot; |
| 518 | request->state = ACTIVE; |
| 519 | } |
| 520 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 521 | static void finish_request(struct transfer_request *request) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 522 | { |
| 523 | fchmod(request->local, 0444); |
| 524 | close(request->local); |
| 525 | |
| 526 | if (request->http_code == 416) { |
| 527 | fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n"); |
| 528 | } else if (request->curl_result != CURLE_OK) { |
| 529 | return; |
| 530 | } |
| 531 | |
| 532 | inflateEnd(&request->stream); |
| 533 | SHA1_Final(request->real_sha1, &request->c); |
| 534 | if (request->zret != Z_STREAM_END) { |
| 535 | unlink(request->tmpfile); |
| 536 | return; |
| 537 | } |
| 538 | if (memcmp(request->sha1, request->real_sha1, 20)) { |
| 539 | unlink(request->tmpfile); |
| 540 | return; |
| 541 | } |
| 542 | request->rename = |
| Junio C Hamano | b721e01 | 2005-10-11 06:22:01 | [diff] [blame] | 543 | move_temp_to_file(request->tmpfile, request->filename); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 544 | |
| 545 | if (request->rename == 0) |
| 546 | pull_say("got %s\n", sha1_to_hex(request->sha1)); |
| 547 | } |
| 548 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 549 | static void release_request(struct transfer_request *request) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 550 | { |
| 551 | struct transfer_request *entry = request_queue_head; |
| 552 | |
| 553 | if (request == request_queue_head) { |
| 554 | request_queue_head = request->next; |
| 555 | } else { |
| 556 | while (entry->next != NULL && entry->next != request) |
| 557 | entry = entry->next; |
| 558 | if (entry->next == request) |
| 559 | entry->next = entry->next->next; |
| 560 | } |
| 561 | |
| 562 | free(request->url); |
| 563 | free(request); |
| 564 | } |
| 565 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 566 | #ifdef USE_CURL_MULTI |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 567 | void process_curl_messages(void) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 568 | { |
| 569 | int num_messages; |
| 570 | struct active_request_slot *slot; |
| 571 | struct transfer_request *request = NULL; |
| 572 | CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages); |
| 573 | |
| 574 | while (curl_message != NULL) { |
| 575 | if (curl_message->msg == CURLMSG_DONE) { |
| 576 | slot = active_queue_head; |
| 577 | while (slot != NULL && |
| 578 | slot->curl != curl_message->easy_handle) |
| 579 | slot = slot->next; |
| 580 | if (slot != NULL) { |
| 581 | curl_multi_remove_handle(curlm, slot->curl); |
| 582 | active_requests--; |
| 583 | slot->done = 1; |
| 584 | slot->in_use = 0; |
| 585 | slot->curl_result = curl_message->data.result; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 586 | curl_easy_getinfo(slot->curl, |
| 587 | CURLINFO_HTTP_CODE, |
| 588 | &slot->http_code); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 589 | request = request_queue_head; |
| 590 | while (request != NULL && |
| 591 | request->slot != slot) |
| 592 | request = request->next; |
| 593 | } else { |
| 594 | fprintf(stderr, "Received DONE message for unknown request!\n"); |
| 595 | } |
| 596 | if (request != NULL) { |
| 597 | request->curl_result = |
| 598 | curl_message->data.result; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 599 | request->http_code = slot->http_code; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 600 | request->slot = NULL; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 601 | request->state = COMPLETE; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 602 | |
| 603 | /* Use alternates if necessary */ |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 604 | if (request->http_code == 404) { |
| 605 | fetch_alternates(alt->base); |
| 606 | if (request->repo->next != NULL) { |
| 607 | request->repo = |
| 608 | request->repo->next; |
| 609 | start_request(request); |
| 610 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 611 | } else { |
| 612 | finish_request(request); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 613 | } |
| 614 | } |
| 615 | } else { |
| 616 | fprintf(stderr, "Unknown CURL message received: %d\n", |
| 617 | (int)curl_message->msg); |
| 618 | } |
| 619 | curl_message = curl_multi_info_read(curlm, &num_messages); |
| 620 | } |
| 621 | } |
| 622 | |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 623 | void process_request_queue(void) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 624 | { |
| 625 | struct transfer_request *request = request_queue_head; |
| Nick Hengeveld | f1a906a | 2005-10-21 19:06:10 | [diff] [blame] | 626 | struct active_request_slot *slot = active_queue_head; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 627 | int num_transfers; |
| 628 | |
| 629 | while (active_requests < max_requests && request != NULL) { |
| 630 | if (request->state == WAITING) { |
| Nick Hengeveld | 11f0daf | 2005-10-11 06:22:01 | [diff] [blame] | 631 | if (has_sha1_file(request->sha1)) |
| 632 | release_request(request); |
| 633 | else |
| 634 | start_request(request); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 635 | curl_multi_perform(curlm, &num_transfers); |
| 636 | } |
| 637 | request = request->next; |
| 638 | } |
| Nick Hengeveld | f1a906a | 2005-10-21 19:06:10 | [diff] [blame] | 639 | |
| 640 | while (slot != NULL) { |
| 641 | if (!slot->in_use && slot->curl != NULL) { |
| 642 | curl_easy_cleanup(slot->curl); |
| 643 | slot->curl = NULL; |
| 644 | } |
| 645 | slot = slot->next; |
| 646 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 647 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 648 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 649 | |
| 650 | void prefetch(unsigned char *sha1) |
| 651 | { |
| 652 | struct transfer_request *newreq; |
| 653 | struct transfer_request *tail; |
| 654 | char *filename = sha1_file_name(sha1); |
| 655 | |
| 656 | newreq = xmalloc(sizeof(*newreq)); |
| 657 | memcpy(newreq->sha1, sha1, 20); |
| 658 | newreq->repo = alt; |
| 659 | newreq->url = NULL; |
| 660 | newreq->local = -1; |
| 661 | newreq->state = WAITING; |
| 662 | snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename); |
| 663 | snprintf(newreq->tmpfile, sizeof(newreq->tmpfile), |
| 664 | "%s.temp", filename); |
| 665 | newreq->next = NULL; |
| 666 | |
| 667 | if (request_queue_head == NULL) { |
| 668 | request_queue_head = newreq; |
| 669 | } else { |
| 670 | tail = request_queue_head; |
| 671 | while (tail->next != NULL) { |
| 672 | tail = tail->next; |
| 673 | } |
| 674 | tail->next = newreq; |
| 675 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 676 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 677 | process_request_queue(); |
| 678 | process_curl_messages(); |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 679 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 680 | } |
| 681 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 682 | static int fetch_index(struct alt_base *repo, unsigned char *sha1) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 683 | { |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 684 | char *hex = sha1_to_hex(sha1); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 685 | char *filename; |
| 686 | char *url; |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 687 | char tmpfile[PATH_MAX]; |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 688 | long prev_posn = 0; |
| 689 | char range[RANGE_HEADER_SIZE]; |
| 690 | struct curl_slist *range_header = NULL; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 691 | |
| 692 | FILE *indexfile; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 693 | struct active_request_slot *slot; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 694 | |
| 695 | if (has_pack_index(sha1)) |
| 696 | return 0; |
| 697 | |
| 698 | if (get_verbosely) |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 699 | fprintf(stderr, "Getting index for pack %s\n", hex); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 700 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 701 | url = xmalloc(strlen(repo->base) + 64); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 702 | sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 703 | |
| 704 | filename = sha1_pack_index_name(sha1); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 705 | snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); |
| 706 | indexfile = fopen(tmpfile, "a"); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 707 | if (!indexfile) |
| 708 | return error("Unable to open local file %s for pack index", |
| 709 | filename); |
| 710 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 711 | slot = get_active_slot(); |
| 712 | curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); |
| 713 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); |
| 714 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 715 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 716 | slot->local = indexfile; |
| 717 | |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 718 | /* If there is data present from a previous transfer attempt, |
| 719 | resume where it left off */ |
| 720 | prev_posn = ftell(indexfile); |
| 721 | if (prev_posn>0) { |
| 722 | if (get_verbosely) |
| 723 | fprintf(stderr, |
| 724 | "Resuming fetch of index for pack %s at byte %ld\n", |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 725 | hex, prev_posn); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 726 | sprintf(range, "Range: bytes=%ld-", prev_posn); |
| 727 | range_header = curl_slist_append(range_header, range); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 728 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 729 | } |
| 730 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 731 | if (start_active_slot(slot)) { |
| 732 | run_active_slot(slot); |
| 733 | if (slot->curl_result != CURLE_OK) { |
| 734 | fclose(indexfile); |
| 735 | return error("Unable to get pack index %s\n%s", url, |
| 736 | curl_errorstr); |
| 737 | } |
| 738 | } else { |
| 739 | return error("Unable to start request"); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 740 | } |
| 741 | |
| 742 | fclose(indexfile); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 743 | |
| Junio C Hamano | b721e01 | 2005-10-11 06:22:01 | [diff] [blame] | 744 | return move_temp_to_file(tmpfile, filename); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 745 | } |
| 746 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 747 | static int setup_index(struct alt_base *repo, unsigned char *sha1) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 748 | { |
| 749 | struct packed_git *new_pack; |
| 750 | if (has_pack_file(sha1)) |
| 751 | return 0; // don't list this as something we can get |
| 752 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 753 | if (fetch_index(repo, sha1)) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 754 | return -1; |
| 755 | |
| 756 | new_pack = parse_pack_index(sha1); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 757 | new_pack->next = repo->packs; |
| 758 | repo->packs = new_pack; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 759 | return 0; |
| 760 | } |
| 761 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 762 | static int fetch_alternates(char *base) |
| 763 | { |
| 764 | int ret = 0; |
| 765 | struct buffer buffer; |
| 766 | char *url; |
| 767 | char *data; |
| 768 | int i = 0; |
| Daniel Barkalow | 1b0c1e6 | 2005-09-18 18:14:19 | [diff] [blame] | 769 | int http_specific = 1; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 770 | struct alt_base *tail = alt; |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 771 | static const char null_byte = '\0'; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 772 | |
| 773 | struct active_request_slot *slot; |
| Junio C Hamano | dc1b5ea | 2005-10-11 06:22:02 | [diff] [blame] | 774 | |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 775 | if (got_alternates) |
| 776 | return 0; |
| 777 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 778 | data = xmalloc(4096); |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 779 | buffer.size = 4096; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 780 | buffer.posn = 0; |
| 781 | buffer.buffer = data; |
| 782 | |
| 783 | if (get_verbosely) |
| 784 | fprintf(stderr, "Getting alternates list\n"); |
| 785 | |
| 786 | url = xmalloc(strlen(base) + 31); |
| 787 | sprintf(url, "%s/objects/info/http-alternates", base); |
| 788 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 789 | slot = get_active_slot(); |
| 790 | curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 791 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, |
| 792 | fwrite_buffer_dynamic); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 793 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| 794 | if (start_active_slot(slot)) { |
| 795 | run_active_slot(slot); |
| 796 | if (slot->curl_result != CURLE_OK || !buffer.posn) { |
| 797 | http_specific = 0; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 798 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 799 | sprintf(url, "%s/objects/info/alternates", base); |
| Daniel Barkalow | 1b0c1e6 | 2005-09-18 18:14:19 | [diff] [blame] | 800 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 801 | slot = get_active_slot(); |
| 802 | curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); |
| 803 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 804 | fwrite_buffer_dynamic); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 805 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| 806 | if (start_active_slot(slot)) { |
| 807 | run_active_slot(slot); |
| 808 | if (slot->curl_result != CURLE_OK) { |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 809 | free(buffer.buffer); |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 810 | if (slot->http_code == 404) |
| 811 | got_alternates = 1; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 812 | return 0; |
| 813 | } |
| 814 | } |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 815 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 816 | } else { |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 817 | free(buffer.buffer); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 818 | return 0; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 819 | } |
| 820 | |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 821 | fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer); |
| 822 | buffer.posn--; |
| 823 | data = buffer.buffer; |
| Daniel Barkalow | 1b0c1e6 | 2005-09-18 18:14:19 | [diff] [blame] | 824 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 825 | while (i < buffer.posn) { |
| 826 | int posn = i; |
| 827 | while (posn < buffer.posn && data[posn] != '\n') |
| 828 | posn++; |
| 829 | if (data[posn] == '\n') { |
| Daniel Barkalow | 1b0c1e6 | 2005-09-18 18:14:19 | [diff] [blame] | 830 | int okay = 0; |
| 831 | int serverlen = 0; |
| 832 | struct alt_base *newalt; |
| 833 | char *target = NULL; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 834 | if (data[i] == '/') { |
| Daniel Barkalow | 1b0c1e6 | 2005-09-18 18:14:19 | [diff] [blame] | 835 | serverlen = strchr(base + 8, '/') - base; |
| 836 | okay = 1; |
| 837 | } else if (!memcmp(data + i, "../", 3)) { |
| 838 | i += 3; |
| 839 | serverlen = strlen(base); |
| 840 | while (i + 2 < posn && |
| 841 | !memcmp(data + i, "../", 3)) { |
| 842 | do { |
| 843 | serverlen--; |
| 844 | } while (serverlen && |
| 845 | base[serverlen - 1] != '/'); |
| 846 | i += 3; |
| 847 | } |
| 848 | // If the server got removed, give up. |
| 849 | okay = strchr(base, ':') - base + 3 < |
| 850 | serverlen; |
| 851 | } else if (http_specific) { |
| 852 | char *colon = strchr(data + i, ':'); |
| 853 | char *slash = strchr(data + i, '/'); |
| 854 | if (colon && slash && colon < data + posn && |
| 855 | slash < data + posn && colon < slash) { |
| 856 | okay = 1; |
| 857 | } |
| 858 | } |
| 859 | // skip 'objects' at end |
| 860 | if (okay) { |
| 861 | target = xmalloc(serverlen + posn - i - 6); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 862 | strncpy(target, base, serverlen); |
| 863 | strncpy(target + serverlen, data + i, |
| 864 | posn - i - 7); |
| 865 | target[serverlen + posn - i - 7] = '\0'; |
| 866 | if (get_verbosely) |
| 867 | fprintf(stderr, |
| 868 | "Also look at %s\n", target); |
| 869 | newalt = xmalloc(sizeof(*newalt)); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 870 | newalt->next = NULL; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 871 | newalt->base = target; |
| 872 | newalt->got_indices = 0; |
| 873 | newalt->packs = NULL; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 874 | while (tail->next != NULL) |
| 875 | tail = tail->next; |
| 876 | tail->next = newalt; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 877 | ret++; |
| 878 | } |
| 879 | } |
| 880 | i = posn + 1; |
| 881 | } |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 882 | |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 883 | got_alternates = 1; |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 884 | free(buffer.buffer); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 885 | return ret; |
| 886 | } |
| 887 | |
| 888 | static int fetch_indices(struct alt_base *repo) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 889 | { |
| 890 | unsigned char sha1[20]; |
| 891 | char *url; |
| 892 | struct buffer buffer; |
| 893 | char *data; |
| 894 | int i = 0; |
| 895 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 896 | struct active_request_slot *slot; |
| 897 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 898 | if (repo->got_indices) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 899 | return 0; |
| 900 | |
| 901 | data = xmalloc(4096); |
| 902 | buffer.size = 4096; |
| 903 | buffer.posn = 0; |
| 904 | buffer.buffer = data; |
| 905 | |
| 906 | if (get_verbosely) |
| 907 | fprintf(stderr, "Getting pack list\n"); |
| 908 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 909 | url = xmalloc(strlen(repo->base) + 21); |
| 910 | sprintf(url, "%s/objects/info/packs", repo->base); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 911 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 912 | slot = get_active_slot(); |
| 913 | curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 914 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, |
| 915 | fwrite_buffer_dynamic); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 916 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| 917 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); |
| 918 | if (start_active_slot(slot)) { |
| 919 | run_active_slot(slot); |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 920 | if (slot->curl_result != CURLE_OK) { |
| 921 | free(buffer.buffer); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 922 | return error("%s", curl_errorstr); |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 923 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 924 | } else { |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 925 | free(buffer.buffer); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 926 | return error("Unable to start request"); |
| 927 | } |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 928 | |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 929 | data = buffer.buffer; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 930 | while (i < buffer.posn) { |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 931 | switch (data[i]) { |
| 932 | case 'P': |
| 933 | i++; |
| 934 | if (i + 52 < buffer.posn && |
| 935 | !strncmp(data + i, " pack-", 6) && |
| 936 | !strncmp(data + i + 46, ".pack\n", 6)) { |
| 937 | get_sha1_hex(data + i + 6, sha1); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 938 | setup_index(repo, sha1); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 939 | i += 51; |
| 940 | break; |
| 941 | } |
| 942 | default: |
| 943 | while (data[i] != '\n') |
| 944 | i++; |
| 945 | } |
| 946 | i++; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 947 | } |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 948 | |
| Sergey Vlasov | bc8f265 | 2005-10-13 17:49:53 | [diff] [blame] | 949 | free(buffer.buffer); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 950 | repo->got_indices = 1; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 951 | return 0; |
| 952 | } |
| 953 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 954 | static int fetch_pack(struct alt_base *repo, unsigned char *sha1) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 955 | { |
| 956 | char *url; |
| 957 | struct packed_git *target; |
| 958 | struct packed_git **lst; |
| 959 | FILE *packfile; |
| 960 | char *filename; |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 961 | char tmpfile[PATH_MAX]; |
| 962 | int ret; |
| 963 | long prev_posn = 0; |
| 964 | char range[RANGE_HEADER_SIZE]; |
| 965 | struct curl_slist *range_header = NULL; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 966 | |
| 967 | struct active_request_slot *slot; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 968 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 969 | if (fetch_indices(repo)) |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 970 | return -1; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 971 | target = find_sha1_pack(sha1, repo->packs); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 972 | if (!target) |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 973 | return -1; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 974 | |
| 975 | if (get_verbosely) { |
| 976 | fprintf(stderr, "Getting pack %s\n", |
| 977 | sha1_to_hex(target->sha1)); |
| 978 | fprintf(stderr, " which contains %s\n", |
| 979 | sha1_to_hex(sha1)); |
| 980 | } |
| 981 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 982 | url = xmalloc(strlen(repo->base) + 65); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 983 | sprintf(url, "%s/objects/pack/pack-%s.pack", |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 984 | repo->base, sha1_to_hex(target->sha1)); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 985 | |
| 986 | filename = sha1_pack_name(target->sha1); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 987 | snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); |
| 988 | packfile = fopen(tmpfile, "a"); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 989 | if (!packfile) |
| 990 | return error("Unable to open local file %s for pack", |
| 991 | filename); |
| 992 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 993 | slot = get_active_slot(); |
| 994 | curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); |
| 995 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); |
| 996 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 997 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 998 | slot->local = packfile; |
| Nick Hengeveld | 1ddea77 | 2005-09-26 17:52:11 | [diff] [blame] | 999 | |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1000 | /* If there is data present from a previous transfer attempt, |
| 1001 | resume where it left off */ |
| 1002 | prev_posn = ftell(packfile); |
| 1003 | if (prev_posn>0) { |
| 1004 | if (get_verbosely) |
| 1005 | fprintf(stderr, |
| 1006 | "Resuming fetch of pack %s at byte %ld\n", |
| 1007 | sha1_to_hex(target->sha1), prev_posn); |
| 1008 | sprintf(range, "Range: bytes=%ld-", prev_posn); |
| 1009 | range_header = curl_slist_append(range_header, range); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1010 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1011 | } |
| 1012 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1013 | if (start_active_slot(slot)) { |
| 1014 | run_active_slot(slot); |
| 1015 | if (slot->curl_result != CURLE_OK) { |
| 1016 | fclose(packfile); |
| 1017 | return error("Unable to get pack file %s\n%s", url, |
| 1018 | curl_errorstr); |
| 1019 | } |
| 1020 | } else { |
| 1021 | return error("Unable to start request"); |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 1022 | } |
| 1023 | |
| 1024 | fclose(packfile); |
| 1025 | |
| Junio C Hamano | b721e01 | 2005-10-11 06:22:01 | [diff] [blame] | 1026 | ret = move_temp_to_file(tmpfile, filename); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1027 | if (ret) |
| Junio C Hamano | b721e01 | 2005-10-11 06:22:01 | [diff] [blame] | 1028 | return ret; |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1029 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1030 | lst = &repo->packs; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 1031 | while (*lst != target) |
| 1032 | lst = &((*lst)->next); |
| 1033 | *lst = (*lst)->next; |
| 1034 | |
| Junio C Hamano | 271421c | 2005-09-30 07:07:39 | [diff] [blame] | 1035 | if (verify_pack(target, 0)) |
| 1036 | return -1; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 1037 | install_packed_git(target); |
| 1038 | |
| 1039 | return 0; |
| 1040 | } |
| 1041 | |
| Peter Hagervall | a7928f8 | 2005-09-28 12:04:54 | [diff] [blame] | 1042 | static int fetch_object(struct alt_base *repo, unsigned char *sha1) |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1043 | { |
| 1044 | char *hex = sha1_to_hex(sha1); |
| Junio C Hamano | 09d9208 | 2005-09-21 09:46:32 | [diff] [blame] | 1045 | int ret; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1046 | struct transfer_request *request = request_queue_head; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1047 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1048 | while (request != NULL && memcmp(request->sha1, sha1, 20)) |
| 1049 | request = request->next; |
| 1050 | if (request == NULL) |
| 1051 | return error("Couldn't find request for %s in the queue", hex); |
| Junio C Hamano | 271421c | 2005-09-30 07:07:39 | [diff] [blame] | 1052 | |
| Nick Hengeveld | 11f0daf | 2005-10-11 06:22:01 | [diff] [blame] | 1053 | if (has_sha1_file(request->sha1)) { |
| 1054 | release_request(request); |
| 1055 | return 0; |
| 1056 | } |
| 1057 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1058 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1059 | while (request->state == WAITING) { |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 1060 | int num_transfers; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1061 | curl_multi_perform(curlm, &num_transfers); |
| 1062 | if (num_transfers < active_requests) { |
| 1063 | process_curl_messages(); |
| 1064 | process_request_queue(); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1065 | } |
| 1066 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1067 | #else |
| 1068 | start_request(request); |
| 1069 | #endif |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1070 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1071 | while (request->state == ACTIVE) { |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1072 | run_active_slot(request->slot); |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1073 | #ifndef USE_CURL_MULTI |
| 1074 | request->curl_result = request->slot->curl_result; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 1075 | request->http_code = request->slot->http_code; |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1076 | request->slot = NULL; |
| 1077 | |
| 1078 | /* Use alternates if necessary */ |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 1079 | if (request->http_code == 404) { |
| 1080 | fetch_alternates(alt->base); |
| 1081 | if (request->repo->next != NULL) { |
| 1082 | request->repo = request->repo->next; |
| 1083 | start_request(request); |
| 1084 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1085 | } else { |
| 1086 | finish_request(request); |
| 1087 | request->state = COMPLETE; |
| 1088 | } |
| 1089 | #endif |
| 1090 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1091 | |
| 1092 | if (request->state == ABORTED) { |
| 1093 | release_request(request); |
| 1094 | return error("Request for %s aborted", hex); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1095 | } |
| 1096 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1097 | if (request->curl_result != CURLE_OK && request->http_code != 416) { |
| Petr Baudis | e2029eb | 2005-10-21 16:18:46 | [diff] [blame] | 1098 | if (request->http_code == 404) |
| 1099 | ret = -1; /* Be silent, it is probably in a pack. */ |
| 1100 | else |
| 1101 | ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", |
| 1102 | request->errorstr, request->curl_result, |
| 1103 | request->http_code, hex); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1104 | release_request(request); |
| 1105 | return ret; |
| barkalow@iabervon.org | 182005b | 2005-08-01 00:54:17 | [diff] [blame] | 1106 | } |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1107 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1108 | if (request->zret != Z_STREAM_END) { |
| 1109 | ret = error("File %s (%s) corrupt\n", hex, request->url); |
| 1110 | release_request(request); |
| 1111 | return ret; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1112 | } |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1113 | |
| 1114 | if (memcmp(request->sha1, request->real_sha1, 20)) { |
| 1115 | release_request(request); |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1116 | return error("File %s has bad hash\n", hex); |
| 1117 | } |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1118 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1119 | if (request->rename < 0) { |
| 1120 | ret = error("unable to write sha1 filename %s: %s", |
| 1121 | request->filename, |
| 1122 | strerror(request->rename)); |
| 1123 | release_request(request); |
| 1124 | return ret; |
| 1125 | } |
| 1126 | |
| 1127 | release_request(request); |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1128 | return 0; |
| 1129 | } |
| 1130 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1131 | int fetch(unsigned char *sha1) |
| 1132 | { |
| 1133 | struct alt_base *altbase = alt; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1134 | |
| 1135 | if (!fetch_object(altbase, sha1)) |
| 1136 | return 0; |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1137 | while (altbase) { |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1138 | if (!fetch_pack(altbase, sha1)) |
| 1139 | return 0; |
| Nick Hengeveld | f7eb290 | 2005-10-21 19:06:20 | [diff] [blame] | 1140 | fetch_alternates(alt->base); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1141 | altbase = altbase->next; |
| 1142 | } |
| 1143 | return error("Unable to find %s under %s\n", sha1_to_hex(sha1), |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1144 | alt->base); |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1145 | } |
| 1146 | |
| Junio C Hamano | 94fa447 | 2005-10-11 06:22:02 | [diff] [blame] | 1147 | static inline int needs_quote(int ch) |
| 1148 | { |
| 1149 | switch (ch) { |
| 1150 | case '/': case '-': case '.': |
| 1151 | case 'A'...'Z': case 'a'...'z': case '0'...'9': |
| 1152 | return 0; |
| 1153 | default: |
| 1154 | return 1; |
| 1155 | } |
| 1156 | } |
| 1157 | |
| 1158 | static inline int hex(int v) |
| 1159 | { |
| 1160 | if (v < 10) return '0' + v; |
| 1161 | else return 'A' + v - 10; |
| 1162 | } |
| 1163 | |
| 1164 | static char *quote_ref_url(const char *base, const char *ref) |
| 1165 | { |
| 1166 | const char *cp; |
| 1167 | char *dp, *qref; |
| 1168 | int len, baselen, ch; |
| 1169 | |
| 1170 | baselen = strlen(base); |
| 1171 | len = baselen + 6; /* "refs/" + NUL */ |
| 1172 | for (cp = ref; (ch = *cp) != 0; cp++, len++) |
| 1173 | if (needs_quote(ch)) |
| 1174 | len += 2; /* extra two hex plus replacement % */ |
| 1175 | qref = xmalloc(len); |
| 1176 | memcpy(qref, base, baselen); |
| 1177 | memcpy(qref + baselen, "refs/", 5); |
| 1178 | for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) { |
| 1179 | if (needs_quote(ch)) { |
| 1180 | *dp++ = '%'; |
| 1181 | *dp++ = hex((ch >> 4) & 0xF); |
| 1182 | *dp++ = hex(ch & 0xF); |
| 1183 | } |
| 1184 | else |
| 1185 | *dp++ = ch; |
| 1186 | } |
| 1187 | *dp = 0; |
| 1188 | |
| 1189 | return qref; |
| 1190 | } |
| 1191 | |
| Daniel Barkalow | cd541a6 | 2005-06-06 20:38:26 | [diff] [blame] | 1192 | int fetch_ref(char *ref, unsigned char *sha1) |
| 1193 | { |
| Junio C Hamano | 94fa447 | 2005-10-11 06:22:02 | [diff] [blame] | 1194 | char *url; |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 1195 | char hex[42]; |
| 1196 | struct buffer buffer; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1197 | char *base = alt->base; |
| 1198 | struct active_request_slot *slot; |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 1199 | buffer.size = 41; |
| 1200 | buffer.posn = 0; |
| 1201 | buffer.buffer = hex; |
| 1202 | hex[41] = '\0'; |
| 1203 | |
| Junio C Hamano | 94fa447 | 2005-10-11 06:22:02 | [diff] [blame] | 1204 | url = quote_ref_url(base, ref); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1205 | slot = get_active_slot(); |
| 1206 | curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); |
| 1207 | curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); |
| 1208 | curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); |
| 1209 | curl_easy_setopt(slot->curl, CURLOPT_URL, url); |
| 1210 | if (start_active_slot(slot)) { |
| 1211 | run_active_slot(slot); |
| 1212 | if (slot->curl_result != CURLE_OK) |
| 1213 | return error("Couldn't get %s for %s\n%s", |
| 1214 | url, ref, curl_errorstr); |
| 1215 | } else { |
| 1216 | return error("Unable to start request"); |
| 1217 | } |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 1218 | |
| 1219 | hex[40] = '\0'; |
| 1220 | get_sha1_hex(hex, sha1); |
| 1221 | return 0; |
| Daniel Barkalow | cd541a6 | 2005-06-06 20:38:26 | [diff] [blame] | 1222 | } |
| 1223 | |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1224 | int main(int argc, char **argv) |
| 1225 | { |
| 1226 | char *commit_id; |
| 1227 | char *url; |
| 1228 | int arg = 1; |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1229 | struct active_request_slot *slot; |
| Nick Hengeveld | e0004e2 | 2005-10-19 21:27:01 | [diff] [blame] | 1230 | char *low_speed_limit; |
| 1231 | char *low_speed_time; |
| Nick Hengeveld | 7b9ae53 | 2005-10-21 19:06:27 | [diff] [blame] | 1232 | char *wait_url; |
| 1233 | int rc = 0; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1234 | |
| 1235 | while (arg < argc && argv[arg][0] == '-') { |
| 1236 | if (argv[arg][1] == 't') { |
| Daniel Barkalow | 4250a5e | 2005-04-30 23:53:56 | [diff] [blame] | 1237 | get_tree = 1; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1238 | } else if (argv[arg][1] == 'c') { |
| Daniel Barkalow | 4250a5e | 2005-04-30 23:53:56 | [diff] [blame] | 1239 | get_history = 1; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1240 | } else if (argv[arg][1] == 'a') { |
| Daniel Barkalow | 4250a5e | 2005-04-30 23:53:56 | [diff] [blame] | 1241 | get_all = 1; |
| 1242 | get_tree = 1; |
| 1243 | get_history = 1; |
| Junio C Hamano | e78d977 | 2005-05-06 08:37:21 | [diff] [blame] | 1244 | } else if (argv[arg][1] == 'v') { |
| 1245 | get_verbosely = 1; |
| Daniel Barkalow | fa3e065 | 2005-06-22 00:45:49 | [diff] [blame] | 1246 | } else if (argv[arg][1] == 'w') { |
| 1247 | write_ref = argv[arg + 1]; |
| 1248 | arg++; |
| Daniel Barkalow | 820eca6 | 2005-09-27 01:38:08 | [diff] [blame] | 1249 | } else if (!strcmp(argv[arg], "--recover")) { |
| 1250 | get_recover = 1; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1251 | } |
| 1252 | arg++; |
| 1253 | } |
| 1254 | if (argc < arg + 2) { |
| Junio C Hamano | 215a7ad | 2005-09-08 00:26:23 | [diff] [blame] | 1255 | usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url"); |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1256 | return 1; |
| 1257 | } |
| 1258 | commit_id = argv[arg]; |
| 1259 | url = argv[arg + 1]; |
| 1260 | |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1261 | curl_global_init(CURL_GLOBAL_ALL); |
| 1262 | |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1263 | #ifdef USE_CURL_MULTI |
| Peter Hagervall | f5dce80 | 2005-10-16 07:01:08 | [diff] [blame] | 1264 | { |
| 1265 | char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); |
| 1266 | if (http_max_requests != NULL) |
| 1267 | max_requests = atoi(http_max_requests); |
| 1268 | } |
| Nick Hengeveld | 3807923 | 2005-10-11 06:22:01 | [diff] [blame] | 1269 | |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1270 | curlm = curl_multi_init(); |
| 1271 | if (curlm == NULL) { |
| 1272 | fprintf(stderr, "Error creating curl multi handle.\n"); |
| 1273 | return 1; |
| 1274 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1275 | #endif |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 1276 | |
| 1277 | if (getenv("GIT_SSL_NO_VERIFY")) |
| 1278 | curl_ssl_verify = 0; |
| 1279 | |
| 1280 | ssl_cert = getenv("GIT_SSL_CERT"); |
| 1281 | #if LIBCURL_VERSION_NUM >= 0x070902 |
| 1282 | ssl_key = getenv("GIT_SSL_KEY"); |
| 1283 | #endif |
| 1284 | #if LIBCURL_VERSION_NUM >= 0x070908 |
| 1285 | ssl_capath = getenv("GIT_SSL_CAPATH"); |
| 1286 | #endif |
| 1287 | ssl_cainfo = getenv("GIT_SSL_CAINFO"); |
| 1288 | |
| Nick Hengeveld | e0004e2 | 2005-10-19 21:27:01 | [diff] [blame] | 1289 | low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); |
| 1290 | if (low_speed_limit != NULL) |
| 1291 | curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); |
| 1292 | low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); |
| 1293 | if (low_speed_time != NULL) |
| 1294 | curl_low_speed_time = strtol(low_speed_time, NULL, 10); |
| 1295 | |
| Nick Hengeveld | d402d55 | 2005-10-15 00:16:01 | [diff] [blame] | 1296 | git_config(http_options); |
| 1297 | |
| 1298 | if (curl_ssl_verify == -1) |
| 1299 | curl_ssl_verify = 1; |
| 1300 | |
| 1301 | #ifdef USE_CURL_MULTI |
| 1302 | if (max_requests < 1) |
| 1303 | max_requests = DEFAULT_MAX_REQUESTS; |
| 1304 | #endif |
| 1305 | |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 1306 | pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); |
| Sergey Vlasov | 1db69b5 | 2005-09-13 15:38:58 | [diff] [blame] | 1307 | no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); |
| Nick Hengeveld | 49a0f24 | 2005-09-28 17:14:04 | [diff] [blame] | 1308 | no_range_header = curl_slist_append(no_range_header, "Range:"); |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1309 | |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 1310 | #ifndef NO_CURL_EASY_DUPHANDLE |
| 1311 | curl_default = get_curl_handle(); |
| barkalow@iabervon.org | 9f6cf65 | 2005-07-31 19:59:20 | [diff] [blame] | 1312 | #endif |
| Darrin Thompson | 3dcb90f | 2005-07-13 02:12:40 | [diff] [blame] | 1313 | |
| Daniel Barkalow | b366156 | 2005-09-15 03:26:08 | [diff] [blame] | 1314 | alt = xmalloc(sizeof(*alt)); |
| 1315 | alt->base = url; |
| 1316 | alt->got_indices = 0; |
| 1317 | alt->packs = NULL; |
| 1318 | alt->next = NULL; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1319 | |
| Daniel Barkalow | 4250a5e | 2005-04-30 23:53:56 | [diff] [blame] | 1320 | if (pull(commit_id)) |
| Nick Hengeveld | 7b9ae53 | 2005-10-21 19:06:27 | [diff] [blame] | 1321 | rc = 1; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1322 | |
| Nick Hengeveld | 0312600 | 2005-10-11 06:22:01 | [diff] [blame] | 1323 | curl_slist_free_all(pragma_header); |
| Sergey Vlasov | 1db69b5 | 2005-09-13 15:38:58 | [diff] [blame] | 1324 | curl_slist_free_all(no_pragma_header); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1325 | curl_slist_free_all(no_range_header); |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 1326 | #ifndef NO_CURL_EASY_DUPHANDLE |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1327 | curl_easy_cleanup(curl_default); |
| Johannes Schindelin | 7baa3e8 | 2005-10-15 18:10:46 | [diff] [blame] | 1328 | #endif |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1329 | slot = active_queue_head; |
| 1330 | while (slot != NULL) { |
| Nick Hengeveld | 7b9ae53 | 2005-10-21 19:06:27 | [diff] [blame] | 1331 | if (slot->in_use) { |
| 1332 | if (get_verbosely) { |
| 1333 | curl_easy_getinfo(slot->curl, |
| 1334 | CURLINFO_EFFECTIVE_URL, |
| 1335 | &wait_url); |
| 1336 | fprintf(stderr, "Waiting for %s\n", wait_url); |
| 1337 | } |
| 1338 | run_active_slot(slot); |
| 1339 | } |
| Nick Hengeveld | f1a906a | 2005-10-21 19:06:10 | [diff] [blame] | 1340 | if (slot->curl != NULL) |
| 1341 | curl_easy_cleanup(slot->curl); |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1342 | slot = slot->next; |
| 1343 | } |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1344 | #ifdef USE_CURL_MULTI |
| Nick Hengeveld | 1d389ab | 2005-10-11 06:22:01 | [diff] [blame] | 1345 | curl_multi_cleanup(curlm); |
| Nick Hengeveld | a7a8d37 | 2005-10-11 06:22:01 | [diff] [blame] | 1346 | #endif |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1347 | curl_global_cleanup(); |
| Nick Hengeveld | 7b9ae53 | 2005-10-21 19:06:27 | [diff] [blame] | 1348 | return rc; |
| Daniel Barkalow | 6eb7ed5 | 2005-04-24 01:47:23 | [diff] [blame] | 1349 | } |