src/libstack/osrf_prefork.c

   1 #include "osrf_prefork.h"
   2 #include <signal.h>
   3
   4 /* true if we just deleted a child.  This will allow us to make sure we're
   5         not trying to use freed memory */
   6 int child_dead;
   7
   8 int main();
   9 void sigchld_handler( int sig );
  10
  11 int osrf_prefork_run(char* appname) {
  12
  13         if(!appname) fatal_handler("osrf_prefork_run requires an appname to run!");
  14
  15         int maxr = 1000;
  16         int maxc = 10;
  17         int minc = 3;
  18
  19         info_handler("Loading config in osrf_forker for app %s", appname);
  20
  21         jsonObject* max_req = osrf_settings_host_value_object("/apps/%s/unix_config/max_requests", appname);
  22         jsonObject* min_children = osrf_settings_host_value_object("/apps/%s/unix_config/min_children", appname);
  23         jsonObject* max_children = osrf_settings_host_value_object("/apps/%s/unix_config/max_children", appname);
  24
  25         if(!max_req) warning_handler("Max requests not defined, assuming 1000");
  26         else maxr = (int) jsonObjectGetNumber(max_req);
  27
  28         if(!min_children) warning_handler("Min children not defined, assuming 3");
  29         else minc = (int) jsonObjectGetNumber(min_children);
  30
  31         if(!max_children) warning_handler("Max children not defined, assuming 10");
  32         else maxc = (int) jsonObjectGetNumber(max_children);
  33
  34         jsonObjectFree(max_req);
  35         jsonObjectFree(min_children);
  36         jsonObjectFree(max_children);
  37         /* --------------------------------------------------- */
  38
  39         char* resc = va_list_to_string("%s_listener", appname);
  40
  41         if(!osrf_system_bootstrap_client_resc( NULL, NULL, resc ))
  42                 fatal_handler("Unable to bootstrap client for osrf_prefork_run()");
  43         free(resc);
  44
  45         prefork_simple* forker = prefork_simple_init(
  46                 osrfSystemGetTransportClient(), maxr, minc, maxc);
  47
  48         forker->appname = strdup(appname);
  49
  50         if(forker == NULL)
  51                 fatal_handler("osrf_prefork_run() failed to create prefork_simple object");
  52
  53         prefork_launch_children(forker);
  54
  55         osrf_prefork_register_routers(appname);
  56
  57         info_handler("Launching osrf_forker for app %s", appname);
  58         prefork_run(forker);
  59
  60         warning_handler("prefork_run() retuned - how??");
  61         prefork_free(forker);
  62         return 0;
  63
  64 }
  65
  66 void osrf_prefork_register_routers( char* appname ) {
  67
  68         osrfStringArray* arr = osrfNewStringArray(4);
  69
  70         int c = osrfConfigGetValueList( NULL, arr, "/routers/router" );
  71         char* routerName = osrfConfigGetValue( NULL, "/router_name" );
  72         transport_client* client = osrfSystemGetTransportClient();
  73
  74         info_handler("router name is %s and we have %d routers to connect to", routerName, c );
  75
  76         while( c ) {
  77                 char* domain = osrfStringArrayGetString(arr, --c);
  78                 if(domain) {
  79
  80                         char* jid = va_list_to_string( "%s@%s/router", routerName, domain );
  81                         info_handler("Registering with router %s", jid );
  82
  83                         transport_message* msg = message_init("registering", NULL, NULL, jid, NULL );
  84                         message_set_router_info( msg, NULL, NULL, appname, "register", 0 );
  85
  86                         client_send_message( client, msg );
  87                         message_free( msg );
  88                         free(jid);
  89                 }
  90         }
  91
  92         free(routerName);
  93         osrfStringArrayFree(arr);
  94 }
  95
  96 void prefork_child_init_hook(prefork_child* child) {
  97
  98         if(!child) return;
  99         info_handler("Child init hook for child %d", child->pid);
 100         char* resc = va_list_to_string("%s_drone",child->appname);
 101         if(!osrf_system_bootstrap_client_resc( NULL, NULL, resc))
 102                 fatal_handler("Unable to bootstrap client for osrf_prefork_run()");
 103         free(resc);
 104 }
 105
 106 void prefork_child_process_request(prefork_child* child, char* data) {
 107         if(!child && child->connection) return;
 108
 109         /* construct the message from the xml */
 110         debug_handler("Child %d received data from parent:\n%s\n", child->pid, data);
 111         transport_message* msg = new_message_from_xml( data );
 112
 113         osrf_stack_transport_handler(msg, child->appname);
 114 }
 115
 116
 117 prefork_simple*  prefork_simple_init( transport_client* client,
 118                 int max_requests, int min_children, int max_children ) {
 119
 120         if( min_children > max_children )
 121                 fatal_handler( "min_children (%d) is greater "
 122                                 "than max_children (%d)", min_children, max_children );
 123
 124         if( max_children > ABS_MAX_CHILDREN )
 125                 fatal_handler( "max_children (%d) is greater than ABS_MAX_CHILDREN (%d)",
 126                                 max_children, ABS_MAX_CHILDREN );
 127
 128         /* flesh out the struct */
 129         prefork_simple* prefork = (prefork_simple*) safe_malloc(sizeof(prefork_simple));
 130         prefork->max_requests = max_requests;
 131         prefork->min_children = min_children;
 132         prefork->max_children = max_children;
 133         prefork->first_child = NULL;
 134         prefork->connection = client;
 135
 136         return prefork;
 137 }
 138
 139 prefork_child*  launch_child( prefork_simple* forker ) {
 140
 141         pid_t pid;
 142         int data_fd[2];
 143         int status_fd[2];
 144
 145         /* Set up the data pipes and add the child struct to the parent */
 146         if( pipe(data_fd) < 0 ) /* build the data pipe*/
 147                 fatal_handler( "Pipe making error" );
 148
 149         if( pipe(status_fd) < 0 ) /* build the status pipe */
 150                 fatal_handler( "Pipe making error" );
 151
 152         debug_handler( "Pipes: %d %d %d %d", data_fd[0], data_fd[1], status_fd[0], status_fd[1] );
 153         prefork_child* child = prefork_child_init( forker->max_requests, data_fd[0],
 154                         data_fd[1], status_fd[0], status_fd[1] );
 155
 156         child->appname = strdup(forker->appname);
 157
 158
 159         add_prefork_child( forker, child );
 160
 161         if( (pid=fork()) < 0 ) fatal_handler( "Forking Error" );
 162
 163         if( pid > 0 ) {  /* parent */
 164
 165                 signal(SIGCHLD, sigchld_handler);
 166                 (forker->current_num_children)++;
 167                 child->pid = pid;
 168
 169                 info_handler( "Parent launched %d", pid );
 170                 /* *no* child pipe FD's can be closed or the parent will re-use fd's that
 171                         the children are currently using */
 172                 return child;
 173         }
 174
 175         else { /* child */
 176
 177                 debug_handler("I am  new child with read_data_fd = %d and write_status_fd = %d",
 178                         child->read_data_fd, child->write_status_fd );
 179
 180                 child->pid = getpid();
 181                 close( child->write_data_fd );
 182                 close( child->read_status_fd );
 183
 184                 /* do the initing */
 185                 prefork_child_init_hook(child);
 186
 187                 prefork_child_wait( child );
 188                 exit(0); /* just to be sure */
 189          }
 190         return NULL;
 191 }
 192
 193
 194 void prefork_launch_children( prefork_simple* forker ) {
 195         if(!forker) return;
 196         int c = 0;
 197         while( c++ < forker->min_children )
 198                 launch_child( forker );
 199 }
 200
 201
 202 void sigchld_handler( int sig ) {
 203         signal(SIGCHLD, sigchld_handler);
 204         child_dead = 1;
 205 }
 206
 207
 208 void reap_children( prefork_simple* forker ) {
 209
 210         pid_t child_pid;
 211         int status;
 212
 213         while( (child_pid=waitpid(-1,&status,WNOHANG)) > 0)
 214                 del_prefork_child( forker, child_pid );
 215
 216         /* replenish */
 217         while( forker->current_num_children < forker->min_children )
 218                 launch_child( forker );
 219
 220         child_dead = 0;
 221 }
 222
 223 void prefork_run(prefork_simple* forker) {
 224
 225         if( forker->first_child == NULL )
 226                 return;
 227
 228         transport_message* cur_msg = NULL;
 229
 230
 231         while(1) {
 232
 233                 if( forker->first_child == NULL ) {/* no more children */
 234                         warning_handler("No more children..." );
 235                         return;
 236                 }
 237
 238                 debug_handler("Forker going into wait for data...");
 239                 cur_msg = client_recv( forker->connection, -1 );
 240
 241                 //fprintf(stderr, "Got Data %f\n", get_timestamp_millis() );
 242
 243                 if( cur_msg == NULL ) continue;
 244
 245                 int honored = 0;        /* true if we've serviced the request */
 246
 247                 while( ! honored ) {
 248
 249                         check_children( forker );
 250
 251                         debug_handler( "Server received inbound data" );
 252                         int k;
 253                         prefork_child* cur_child = forker->first_child;
 254
 255                         /* Look for an available child */
 256                         for( k = 0; k < forker->current_num_children; k++ ) {
 257
 258                                 debug_handler("Searching for available child. cur_child->pid = %d", cur_child->pid );
 259                                 debug_handler("Current num children %d and loop %d", forker->current_num_children, k);
 260
 261                                 if( cur_child->available ) {
 262                                         debug_handler( "sending data to %d", cur_child->pid );
 263
 264                                         message_prepare_xml( cur_msg );
 265                                         char* data = cur_msg->msg_xml;
 266                                         if( ! data || strlen(data) < 1 ) break;
 267
 268                                         cur_child->available = 0;
 269                                         debug_handler( "Writing to child fd %d", cur_child->write_data_fd );
 270
 271                                         int written = 0;
 272                                         //fprintf(stderr, "Writing Data %f\n", get_timestamp_millis() );
 273                                         if( (written = write( cur_child->write_data_fd, data, strlen(data) + 1 )) < 0 ) {
 274                                                 warning_handler("Write returned error %d", errno);
 275                                                 cur_child = cur_child->next;
 276                                                 continue;
 277                                         }
 278
 279                                         //fprintf(stderr, "Wrote %d bytes to child\n", written);
 280
 281                                         forker->first_child = cur_child->next;
 282                                         honored = 1;
 283                                         break;
 284                                 } else
 285                                         cur_child = cur_child->next;
 286                         }
 287
 288                         /* if none available, add a new child if we can */
 289                         if( ! honored ) {
 290                                 debug_handler("Not enough children, attempting to add...");
 291                                 if( forker->current_num_children < forker->max_children ) {
 292                                         debug_handler( "Launching new child with current_num = %d",
 293                                                         forker->current_num_children );
 294
 295                                         prefork_child* new_child = launch_child( forker );
 296                                         message_prepare_xml( cur_msg );
 297                                         char* data = cur_msg->msg_xml;
 298                                         if( ! data || strlen(data) < 1 ) break;
 299                                         new_child->available = 0;
 300                                         debug_handler( "sending data to %d", new_child->pid );
 301                                         debug_handler( "Writing to new child fd %d", new_child->write_data_fd );
 302                                         write( new_child->write_data_fd, data, strlen(data) + 1 );
 303                                         forker->first_child = new_child->next;
 304                                         honored = 1;
 305                                 }
 306                         }
 307
 308                         if( !honored ) {
 309                                 warning_handler( "No children available, sleeping and looping..." );
 310                                 usleep( 50000 ); /* 50 milliseconds */
 311                         }
 312
 313                         if( child_dead )
 314                                 reap_children(forker);
 315
 316
 317                         //fprintf(stderr, "Parent done with request %f\n", get_timestamp_millis() );
 318
 319                 } // honored?
 320
 321                 message_free( cur_msg );
 322
 323         } /* top level listen loop */
 324
 325 }
 326
 327
 328 void check_children( prefork_simple* forker ) {
 329
 330         //check_begin:
 331
 332         int select_ret;
 333         fd_set read_set;
 334         FD_ZERO(&read_set);
 335         int max_fd = 0;
 336         int n;
 337
 338         struct timeval tv;
 339         tv.tv_sec       = 0;
 340         tv.tv_usec      = 0;
 341
 342         if( child_dead )
 343                 reap_children(forker);
 344
 345         prefork_child* cur_child = forker->first_child;
 346
 347         int i;
 348         for( i = 0; i!= forker->current_num_children; i++ ) {
 349
 350                 if( cur_child->read_status_fd > max_fd )
 351                         max_fd = cur_child->read_status_fd;
 352                 FD_SET( cur_child->read_status_fd, &read_set );
 353                 cur_child = cur_child->next;
 354         }
 355
 356         FD_CLR(0,&read_set);/* just to be sure */
 357
 358         if( (select_ret=select( max_fd + 1 , &read_set, NULL, NULL, &tv)) == -1 ) {
 359                 warning_handler( "Select returned error %d on check_children", errno );
 360         }
 361
 362         if( select_ret == 0 )
 363                 return;
 364
 365         /* see if one of a child has told us it's done */
 366         cur_child = forker->first_child;
 367         int j;
 368         int num_handled = 0;
 369         for( j = 0; j!= forker->current_num_children && num_handled < select_ret ; j++ ) {
 370
 371                 if( FD_ISSET( cur_child->read_status_fd, &read_set ) ) {
 372                         //printf( "Server received status from a child %d\n", cur_child->pid );
 373                         debug_handler( "Server received status from a child %d", cur_child->pid );
 374
 375                         num_handled++;
 376
 377                         /* now suck off the data */
 378                         char buf[64];
 379                         memset( buf, 0, 64);
 380                         if( (n=read(cur_child->read_status_fd, buf, 63))  < 0 ) {
 381                                 warning_handler("Read error afer select in child status read with errno %d", errno);
 382                         }
 383
 384                         debug_handler( "Read %d bytes from status buffer: %s", n, buf );
 385                         cur_child->available = 1;
 386                 }
 387                 cur_child = cur_child->next;
 388         }
 389
 390 }
 391
 392
 393 void prefork_child_wait( prefork_child* child ) {
 394
 395         int i,n;
 396         growing_buffer* gbuf = buffer_init( READ_BUFSIZE );
 397         char buf[READ_BUFSIZE];
 398         memset( buf, 0, READ_BUFSIZE );
 399
 400         for( i = 0; i!= child->max_requests; i++ ) {
 401
 402                 n = -1;
 403                 clr_fl(child->read_data_fd, O_NONBLOCK );
 404                 while( (n=read(child->read_data_fd, buf, READ_BUFSIZE-1)) > 0 ) {
 405                         buffer_add( gbuf, buf );
 406                         memset( buf, 0, READ_BUFSIZE );
 407
 408                         //fprintf(stderr, "Child read %d bytes\n", n);
 409
 410                         if( n == READ_BUFSIZE ) {
 411                                 //fprintf(stderr, "We read READ_BUFSIZE data....\n");
 412                                 /* XXX */
 413                                 /* either we have exactly READ_BUFSIZE data,
 414                                         or there's more waiting that we need to grab*/
 415                                 /* must set to non-block for reading more */
 416                         } else {
 417                                 //fprintf(stderr, "Read Data %f\n", get_timestamp_millis() );
 418                                 prefork_child_process_request(child, gbuf->buf);
 419                                 buffer_reset( gbuf );
 420                                 break;
 421                         }
 422                 }
 423
 424                 if( n < 0 ) {
 425                         warning_handler( "Child read returned error with errno %d", errno );
 426                         break;
 427                 }
 428
 429                 if( i < child->max_requests - 1 )
 430                         write( child->write_status_fd, "available" /*less than 64 bytes*/, 9 );
 431         }
 432
 433         buffer_free(gbuf);
 434
 435         debug_handler("Child exiting...[%d]", getpid() );
 436
 437         exit(0);
 438 }
 439
 440
 441 void add_prefork_child( prefork_simple* forker, prefork_child* child ) {
 442
 443         if( forker->first_child == NULL ) {
 444                 forker->first_child = child;
 445                 child->next = child;
 446                 return;
 447         }
 448
 449         /* we put the child in as the last because, regardless,
 450                 we have to do the DLL splice dance, and this is the
 451            simplest way */
 452
 453         prefork_child* start_child = forker->first_child;
 454         while(1) {
 455                 if( forker->first_child->next == start_child )
 456                         break;
 457                 forker->first_child = forker->first_child->next;
 458         }
 459
 460         /* here we know that forker->first_child is the last element
 461                 in the list and start_child is the first.  Insert the
 462                 new child between them*/
 463
 464         forker->first_child->next = child;
 465         child->next = start_child;
 466         return;
 467 }
 468
 469 prefork_child* find_prefork_child( prefork_simple* forker, pid_t pid ) {
 470
 471         if( forker->first_child == NULL ) { return NULL; }
 472         prefork_child* start_child = forker->first_child;
 473         do {
 474                 if( forker->first_child->pid == pid )
 475                         return forker->first_child;
 476         } while( (forker->first_child = forker->first_child->next) != start_child );
 477
 478         return NULL;
 479 }
 480
 481
 482 void del_prefork_child( prefork_simple* forker, pid_t pid ) {
 483
 484         if( forker->first_child == NULL ) { return; }
 485
 486         (forker->current_num_children)--;
 487         debug_handler("Deleting Child: %d", pid );
 488
 489         prefork_child* start_child = forker->first_child; /* starting point */
 490         prefork_child* cur_child        = start_child; /* current pointer */
 491         prefork_child* prev_child       = start_child; /* the trailing pointer */
 492
 493         /* special case where there is only one in the list */
 494         if( start_child == start_child->next ) {
 495                 if( start_child->pid == pid ) {
 496                         forker->first_child = NULL;
 497
 498                         close( start_child->read_data_fd );
 499                         close( start_child->write_data_fd );
 500                         close( start_child->read_status_fd );
 501                         close( start_child->write_status_fd );
 502
 503                         prefork_child_free( start_child );
 504                 }
 505                 return;
 506         }
 507
 508
 509         /* special case where the first item in the list needs to be removed */
 510         if( start_child->pid == pid ) {
 511
 512                 /* find the last one so we can remove the start_child */
 513                 do {
 514                         prev_child = cur_child;
 515                         cur_child = cur_child->next;
 516                 }while( cur_child != start_child );
 517
 518                 /* now cur_child == start_child */
 519                 prev_child->next = cur_child->next;
 520                 forker->first_child = prev_child;
 521
 522                 close( cur_child->read_data_fd );
 523                 close( cur_child->write_data_fd );
 524                 close( cur_child->read_status_fd );
 525                 close( cur_child->write_status_fd );
 526
 527                 prefork_child_free( cur_child );
 528                 return;
 529         }
 530
 531         do {
 532                 prev_child = cur_child;
 533                 cur_child = cur_child->next;
 534
 535                 if( cur_child->pid == pid ) {
 536                         prev_child->next = cur_child->next;
 537
 538                         close( cur_child->read_data_fd );
 539                         close( cur_child->write_data_fd );
 540                         close( cur_child->read_status_fd );
 541                         close( cur_child->write_status_fd );
 542
 543                         prefork_child_free( cur_child );
 544                         return;
 545                 }
 546
 547         } while(cur_child != start_child);
 548 }
 549
 550
 551
 552
 553 prefork_child* prefork_child_init(
 554         int max_requests, int read_data_fd, int write_data_fd,
 555         int read_status_fd, int write_status_fd ) {
 556
 557         prefork_child* child = (prefork_child*) safe_malloc(sizeof(prefork_child));
 558         child->max_requests             = max_requests;
 559         child->read_data_fd             = read_data_fd;
 560         child->write_data_fd            = write_data_fd;
 561         child->read_status_fd   = read_status_fd;
 562         child->write_status_fd  = write_status_fd;
 563         child->available                        = 1;
 564
 565         return child;
 566 }
 567
 568
 569 int prefork_free( prefork_simple* prefork ) {
 570
 571         while( prefork->first_child != NULL ) {
 572                 info_handler( "Killing children and sleeping 1 to reap..." );
 573                 kill( 0,        SIGKILL );
 574                 sleep(1);
 575         }
 576
 577         client_free(prefork->connection);
 578         free(prefork->appname);
 579         free( prefork );
 580         return 1;
 581 }
 582
 583 int prefork_child_free( prefork_child* child ) {
 584         free(child->appname);
 585         close(child->read_data_fd);
 586         close(child->write_status_fd);
 587         free( child );
 588         return 1;
 589 }
 590