added connection oriented statefull session handling to the server stack
[OpenSRF.git] / src / libstack / osrf_prefork.c
1 #include "osrf_prefork.h"
2 #include <signal.h>
3 #include "osrf_app_session.h"
4 #include "osrf_application.h"
5
6 /* true if we just deleted a child.  This will allow us to make sure we're
7         not trying to use freed memory */
8 int child_dead;
9
10 int main();
11 void sigchld_handler( int sig );
12
13 int osrf_prefork_run(char* appname) {
14
15         if(!appname) {
16                 osrfLogError( OSRF_LOG_MARK, "osrf_prefork_run requires an appname to run!");
17                 return -1;
18         }
19
20         set_proc_title( "OpenSRF Listener [%s]", appname );
21
22         int maxr = 1000; 
23         int maxc = 10;
24         int minc = 3;
25
26         osrfLogInfo( OSRF_LOG_MARK, "Loading config in osrf_forker for app %s", appname);
27
28         jsonObject* max_req = osrf_settings_host_value_object("/apps/%s/unix_config/max_requests", appname);
29         jsonObject* min_children = osrf_settings_host_value_object("/apps/%s/unix_config/min_children", appname);
30         jsonObject* max_children = osrf_settings_host_value_object("/apps/%s/unix_config/max_children", appname);
31
32         char* keepalive = osrf_settings_host_value("/apps/%s/keepalive", appname);
33         time_t kalive;
34         if( keepalive ) {
35                 kalive = atoi(keepalive);
36                 free(keepalive);
37         } else {
38                 kalive = 5; /* give it a default */
39         }
40
41         osrfLogInfo(OSRF_LOG_MARK, "keepalive setting = %d seconds", kalive);
42
43
44         
45         if(!max_req) osrfLogWarning( OSRF_LOG_MARK, "Max requests not defined, assuming 1000");
46         else maxr = (int) jsonObjectGetNumber(max_req);
47
48         if(!min_children) osrfLogWarning( OSRF_LOG_MARK, "Min children not defined, assuming 3");
49         else minc = (int) jsonObjectGetNumber(min_children);
50
51         if(!max_children) osrfLogWarning( OSRF_LOG_MARK, "Max children not defined, assuming 10");
52         else maxc = (int) jsonObjectGetNumber(max_children);
53
54         jsonObjectFree(max_req);
55         jsonObjectFree(min_children);
56         jsonObjectFree(max_children);
57         /* --------------------------------------------------- */
58
59         char* resc = va_list_to_string("%s_listener", appname);
60
61         if(!osrf_system_bootstrap_client_resc( NULL, NULL, resc )) {
62                 osrfLogError( OSRF_LOG_MARK, "Unable to bootstrap client for osrf_prefork_run()");
63                 free(resc);
64                 return -1;
65         }
66
67         free(resc);
68
69         prefork_simple* forker = prefork_simple_init(
70                 osrfSystemGetTransportClient(), maxr, minc, maxc);
71
72         forker->appname = strdup(appname);
73         forker->keepalive       = kalive;
74
75         if(forker == NULL) {
76                 osrfLogError( OSRF_LOG_MARK, "osrf_prefork_run() failed to create prefork_simple object");
77                 return -1;
78         }
79
80         prefork_launch_children(forker);
81
82         osrf_prefork_register_routers(appname);
83         
84         osrfLogInfo( OSRF_LOG_MARK, "Launching osrf_forker for app %s", appname);
85         prefork_run(forker);
86         
87         osrfLogWarning( OSRF_LOG_MARK, "prefork_run() retuned - how??");
88         prefork_free(forker);
89         return 0;
90
91 }
92
93 void osrf_prefork_register_routers( char* appname ) {
94
95         osrfStringArray* arr = osrfNewStringArray(4);
96
97         int c = osrfConfigGetValueList( NULL, arr, "/routers/router" );
98         char* routerName = osrfConfigGetValue( NULL, "/router_name" );
99         transport_client* client = osrfSystemGetTransportClient();
100
101         osrfLogInfo( OSRF_LOG_MARK, "router name is %s and we have %d routers to connect to", routerName, c );
102
103         while( c ) {
104                 char* domain = osrfStringArrayGetString(arr, --c);
105                 if(domain) {
106
107                         char* jid = va_list_to_string( "%s@%s/router", routerName, domain );
108                         osrfLogInfo( OSRF_LOG_MARK, "Registering with router %s", jid );
109
110                         transport_message* msg = message_init("registering", NULL, NULL, jid, NULL );
111                         message_set_router_info( msg, NULL, NULL, appname, "register", 0 );
112
113                         client_send_message( client, msg );
114                         message_free( msg );
115                         free(jid);
116                 }
117         }
118
119         free(routerName);
120         osrfStringArrayFree(arr);
121 }
122
123 void prefork_child_init_hook(prefork_child* child) {
124
125         if(!child) return;
126         osrfLogDebug( OSRF_LOG_MARK, "Child init hook for child %d", child->pid);
127         char* resc = va_list_to_string("%s_drone",child->appname);
128
129         /* we want to remove traces of our parents socket connection 
130          * so we can have our own */
131         osrfSystemIgnoreTransportClient();
132
133         if(!osrf_system_bootstrap_client_resc( NULL, NULL, resc)) {
134                 osrfLogError( OSRF_LOG_MARK, "Unable to bootstrap client for osrf_prefork_run()");
135                 free(resc);
136                 return;
137         }
138
139         free(resc);
140
141         if( ! osrfAppRunChildInit(child->appname) ) {
142                 osrfLogDebug(OSRF_LOG_MARK, "Prefork child_init succeeded\n");
143         } else {
144                 osrfLogError(OSRF_LOG_MARK, "Prefork child_init failed\n");
145         }
146
147         set_proc_title( "OpenSRF Drone [%s]", child->appname );
148 }
149
150 void prefork_child_process_request(prefork_child* child, char* data) {
151         if( !child ) return;
152
153         /* construct the message from the xml */
154         transport_message* msg = new_message_from_xml( data );
155
156         osrfAppSession* session = osrf_stack_transport_handler(msg, child->appname);
157         if(!session) return;
158
159         if( session->stateless && session->state != OSRF_SESSION_CONNECTED ) {
160                 osrfAppSessionFree( session );
161                 return;
162         }
163
164         osrfLogDebug( OSRF_LOG_MARK, "Entering keepalive loop for session %s", session->session_id );
165         int keepalive = child->keepalive;
166         int retval;
167         time_t start;
168         time_t end;
169
170         while(1) {
171
172                 osrfLogDebug(OSRF_LOG_MARK, 
173                                 "osrf_prefork calling queue_wait [%d] in keepalive loop", keepalive);
174                 start           = time(NULL);
175                 retval  = osrf_app_session_queue_wait(session, keepalive);
176                 end             = time(NULL);
177
178                 if(retval) {
179                         osrfLogError(OSRF_LOG_MARK, "queue-wait returned non-success %d", retval);
180                         break;
181                 }
182
183                 /* see if the client disconnected from us */
184                 if(session->state != OSRF_SESSION_CONNECTED) break;
185
186                 /* see if the used up the timeout */
187                 if( (end - start) >= keepalive ) {
188
189                         osrfLogDebug(OSRF_LOG_MARK, "Keepalive timed out, exiting connected session");
190
191                         osrfAppSessionStatus( 
192                                         session, 
193                                         OSRF_STATUS_TIMEOUT, 
194                                         "osrfConnectStatus", 
195                                         0, "Disconnected on timeout" );
196
197                         break;
198                 }
199         }
200
201         osrfLogDebug( OSRF_LOG_MARK, "Exiting keepalive loop for session %s", session->session_id );
202         osrfAppSessionFree( session );
203         return;
204 }
205
206
207 prefork_simple*  prefork_simple_init( transport_client* client, 
208                 int max_requests, int min_children, int max_children ) {
209
210         if( min_children > max_children ) {
211                 osrfLogError( OSRF_LOG_MARK,  "min_children (%d) is greater "
212                                 "than max_children (%d)", min_children, max_children );
213                 return NULL;
214         }
215
216         if( max_children > ABS_MAX_CHILDREN ) {
217                 osrfLogError( OSRF_LOG_MARK,  "max_children (%d) is greater than ABS_MAX_CHILDREN (%d)",
218                                 max_children, ABS_MAX_CHILDREN );
219                 return NULL;
220         }
221
222         osrfLogInfo(OSRF_LOG_MARK, "Prefork launching child with max_request=%d,"
223                 "min_children=%d, max_children=%d", max_requests, min_children, max_children );
224
225         /* flesh out the struct */
226         prefork_simple* prefork = (prefork_simple*) safe_malloc(sizeof(prefork_simple));        
227         prefork->max_requests = max_requests;
228         prefork->min_children = min_children;
229         prefork->max_children = max_children;
230         prefork->first_child = NULL;
231         prefork->connection = client;
232
233         return prefork;
234 }
235
236 prefork_child*  launch_child( prefork_simple* forker ) {
237
238         pid_t pid;
239         int data_fd[2];
240         int status_fd[2];
241
242         /* Set up the data pipes and add the child struct to the parent */
243         if( pipe(data_fd) < 0 ) { /* build the data pipe*/
244                 osrfLogError( OSRF_LOG_MARK,  "Pipe making error" );
245                 return NULL;
246         }
247
248         if( pipe(status_fd) < 0 ) {/* build the status pipe */
249                 osrfLogError( OSRF_LOG_MARK,  "Pipe making error" );
250                 return NULL;
251         }
252
253         osrfLogInternal( OSRF_LOG_MARK,  "Pipes: %d %d %d %d", data_fd[0], data_fd[1], status_fd[0], status_fd[1] );
254         prefork_child* child = prefork_child_init( forker->max_requests, data_fd[0], 
255                         data_fd[1], status_fd[0], status_fd[1] );
256
257         child->appname = strdup(forker->appname);
258         child->keepalive = forker->keepalive;
259
260
261         add_prefork_child( forker, child );
262
263         if( (pid=fork()) < 0 ) {
264                 osrfLogError( OSRF_LOG_MARK,  "Forking Error" );
265                 return NULL;
266         }
267
268         if( pid > 0 ) {  /* parent */
269
270                 signal(SIGCHLD, sigchld_handler);
271                 (forker->current_num_children)++;
272                 child->pid = pid;
273
274                 osrfLogDebug( OSRF_LOG_MARK,  "Parent launched %d", pid );
275                 /* *no* child pipe FD's can be closed or the parent will re-use fd's that
276                         the children are currently using */
277                 return child;
278         }
279
280         else { /* child */
281
282                 osrfLogInternal( OSRF_LOG_MARK, "I am  new child with read_data_fd = %d and write_status_fd = %d",
283                         child->read_data_fd, child->write_status_fd );
284
285                 child->pid = getpid();
286                 close( child->write_data_fd );
287                 close( child->read_status_fd );
288
289                 /* do the initing */
290                 prefork_child_init_hook(child);
291
292                 prefork_child_wait( child );
293                 exit(0); /* just to be sure */
294          }
295         return NULL;
296 }
297
298
299 void prefork_launch_children( prefork_simple* forker ) {
300         if(!forker) return;
301         int c = 0;
302         while( c++ < forker->min_children )
303                 launch_child( forker );
304 }
305
306
307 void sigchld_handler( int sig ) {
308         signal(SIGCHLD, sigchld_handler);
309         child_dead = 1;
310 }
311
312
313 void reap_children( prefork_simple* forker ) {
314
315         pid_t child_pid;
316         int status;
317
318         while( (child_pid=waitpid(-1,&status,WNOHANG)) > 0) 
319                 del_prefork_child( forker, child_pid ); 
320
321         /* replenish */
322         while( forker->current_num_children < forker->min_children ) 
323                 launch_child( forker );
324
325         child_dead = 0;
326 }
327
328 void prefork_run(prefork_simple* forker) {
329
330         if( forker->first_child == NULL )
331                 return;
332
333         transport_message* cur_msg = NULL;
334
335
336         while(1) {
337
338                 if( forker->first_child == NULL ) {/* no more children */
339                         osrfLogWarning( OSRF_LOG_MARK, "No more children..." );
340                         return;
341                 }
342
343                 osrfLogDebug( OSRF_LOG_MARK, "Forker going into wait for data...");
344                 cur_msg = client_recv( forker->connection, -1 );
345
346                 //fprintf(stderr, "Got Data %f\n", get_timestamp_millis() );
347
348                 if( cur_msg == NULL ) continue;
349
350                 int honored = 0;        /* true if we've serviced the request */
351
352                 while( ! honored ) {
353
354                         check_children( forker ); 
355
356                         osrfLogDebug( OSRF_LOG_MARK,  "Server received inbound data" );
357                         int k;
358                         prefork_child* cur_child = forker->first_child;
359
360                         /* Look for an available child */
361                         for( k = 0; k < forker->current_num_children; k++ ) {
362
363                                 osrfLogInternal( OSRF_LOG_MARK, "Searching for available child. cur_child->pid = %d", cur_child->pid );
364                                 osrfLogInternal( OSRF_LOG_MARK, "Current num children %d and loop %d", forker->current_num_children, k);
365                         
366                                 if( cur_child->available ) {
367                                         osrfLogDebug( OSRF_LOG_MARK,  "forker sending data to %d", cur_child->pid );
368
369                                         message_prepare_xml( cur_msg );
370                                         char* data = cur_msg->msg_xml;
371                                         if( ! data || strlen(data) < 1 ) break;
372
373                                         cur_child->available = 0;
374                                         osrfLogInternal( OSRF_LOG_MARK,  "Writing to child fd %d", cur_child->write_data_fd );
375
376                                         int written = 0;
377                                         //fprintf(stderr, "Writing Data %f\n", get_timestamp_millis() );
378                                         if( (written = write( cur_child->write_data_fd, data, strlen(data) + 1 )) < 0 ) {
379                                                 osrfLogWarning( OSRF_LOG_MARK, "Write returned error %d", errno);
380                                                 cur_child = cur_child->next;
381                                                 continue;
382                                         }
383
384                                         //fprintf(stderr, "Wrote %d bytes to child\n", written);
385
386                                         forker->first_child = cur_child->next;
387                                         honored = 1;
388                                         break;
389                                 } else 
390                                         cur_child = cur_child->next;
391                         } 
392
393                         /* if none available, add a new child if we can */
394                         if( ! honored ) {
395                                 osrfLogDebug( OSRF_LOG_MARK, "Not enough children, attempting to add...");
396                                 if( forker->current_num_children < forker->max_children ) {
397                                         osrfLogDebug( OSRF_LOG_MARK,  "Launching new child with current_num = %d",
398                                                         forker->current_num_children );
399
400                                         prefork_child* new_child = launch_child( forker );
401                                         message_prepare_xml( cur_msg );
402                                         char* data = cur_msg->msg_xml;
403                                         if( ! data || strlen(data) < 1 ) break;
404                                         new_child->available = 0;
405                                         osrfLogDebug( OSRF_LOG_MARK,  "Writing to new child fd %d : pid %d", 
406                                                         new_child->write_data_fd, new_child->pid );
407                                         write( new_child->write_data_fd, data, strlen(data) + 1 );
408                                         forker->first_child = new_child->next;
409                                         honored = 1;
410                                 }
411                         }
412
413                         if( !honored ) {
414                                 osrfLogWarning( OSRF_LOG_MARK,  "No children available, sleeping and looping..." );
415                                 usleep( 50000 ); /* 50 milliseconds */
416                         }
417
418                         if( child_dead )
419                                 reap_children(forker);
420
421
422                         //fprintf(stderr, "Parent done with request %f\n", get_timestamp_millis() );
423
424                 } // honored?
425
426                 message_free( cur_msg );
427
428         } /* top level listen loop */
429
430 }
431
432
433 void check_children( prefork_simple* forker ) {
434
435         //check_begin:
436
437         int select_ret;
438         fd_set read_set;
439         FD_ZERO(&read_set);
440         int max_fd = 0;
441         int n;
442
443         struct timeval tv;
444         tv.tv_sec       = 0;
445         tv.tv_usec      = 0;
446
447         if( child_dead )
448                 reap_children(forker);
449
450         prefork_child* cur_child = forker->first_child;
451
452         int i;
453         for( i = 0; i!= forker->current_num_children; i++ ) {
454
455                 if( cur_child->read_status_fd > max_fd )
456                         max_fd = cur_child->read_status_fd;
457                 FD_SET( cur_child->read_status_fd, &read_set );
458                 cur_child = cur_child->next;
459         }
460
461         FD_CLR(0,&read_set);/* just to be sure */
462
463         if( (select_ret=select( max_fd + 1 , &read_set, NULL, NULL, &tv)) == -1 ) {
464                 osrfLogWarning( OSRF_LOG_MARK,  "Select returned error %d on check_children", errno );
465         }
466
467         if( select_ret == 0 )
468                 return;
469
470         /* see if one of a child has told us it's done */
471         cur_child = forker->first_child;
472         int j;
473         int num_handled = 0;
474         for( j = 0; j!= forker->current_num_children && num_handled < select_ret ; j++ ) {
475
476                 if( FD_ISSET( cur_child->read_status_fd, &read_set ) ) {
477                         //printf( "Server received status from a child %d\n", cur_child->pid );
478                         osrfLogDebug( OSRF_LOG_MARK,  "Server received status from a child %d", cur_child->pid );
479
480                         num_handled++;
481
482                         /* now suck off the data */
483                         char buf[64];
484                         memset( buf, 0, 64);
485                         if( (n=read(cur_child->read_status_fd, buf, 63))  < 0 ) {
486                                 osrfLogWarning( OSRF_LOG_MARK, "Read error afer select in child status read with errno %d", errno);
487                         }
488
489                         osrfLogDebug( OSRF_LOG_MARK,  "Read %d bytes from status buffer: %s", n, buf );
490                         cur_child->available = 1;
491                 }
492                 cur_child = cur_child->next;
493         } 
494
495 }
496
497
498 void prefork_child_wait( prefork_child* child ) {
499
500         int i,n;
501         growing_buffer* gbuf = buffer_init( READ_BUFSIZE );
502         char buf[READ_BUFSIZE];
503         memset( buf, 0, READ_BUFSIZE );
504
505         for( i = 0; i < child->max_requests; i++ ) {
506
507                 n = -1;
508                 clr_fl(child->read_data_fd, O_NONBLOCK );
509                 while( (n=read(child->read_data_fd, buf, READ_BUFSIZE-1)) > 0 ) {
510                         buffer_add( gbuf, buf );
511                         memset( buf, 0, READ_BUFSIZE );
512
513                         //fprintf(stderr, "Child read %d bytes\n", n);
514
515                         if( n == READ_BUFSIZE ) { 
516                                 //fprintf(stderr, "We read READ_BUFSIZE data....\n");
517                                 /* XXX */
518                                 /* either we have exactly READ_BUFSIZE data, 
519                                         or there's more waiting that we need to grab*/
520                                 /* must set to non-block for reading more */
521                         } else {
522                                 //fprintf(stderr, "Read Data %f\n", get_timestamp_millis() );
523                                 prefork_child_process_request(child, gbuf->buf);
524                                 buffer_reset( gbuf );
525                                 break;
526                         }
527                 }
528
529                 if( n < 0 ) {
530                         osrfLogWarning( OSRF_LOG_MARK,  "Prefork child read returned error with errno %d", errno );
531                         break;
532                 }
533
534                 if( i < child->max_requests - 1 ) 
535                         write( child->write_status_fd, "available" /*less than 64 bytes*/, 9 );
536         }
537
538         buffer_free(gbuf);
539
540         osrfLogDebug( OSRF_LOG_MARK, "Child with max-requests=%d, num-served=%d exiting...[%d]", 
541                         child->max_requests, i, getpid() );
542
543         exit(0);
544 }
545
546
547 void add_prefork_child( prefork_simple* forker, prefork_child* child ) {
548         
549         if( forker->first_child == NULL ) {
550                 forker->first_child = child;
551                 child->next = child;
552                 return;
553         }
554
555         /* we put the child in as the last because, regardless, 
556                 we have to do the DLL splice dance, and this is the
557            simplest way */
558
559         prefork_child* start_child = forker->first_child;
560         while(1) {
561                 if( forker->first_child->next == start_child ) 
562                         break;
563                 forker->first_child = forker->first_child->next;
564         }
565
566         /* here we know that forker->first_child is the last element 
567                 in the list and start_child is the first.  Insert the
568                 new child between them*/
569
570         forker->first_child->next = child;
571         child->next = start_child;
572         return;
573 }
574
575 prefork_child* find_prefork_child( prefork_simple* forker, pid_t pid ) {
576
577         if( forker->first_child == NULL ) { return NULL; }
578         prefork_child* start_child = forker->first_child;
579         do {
580                 if( forker->first_child->pid == pid ) 
581                         return forker->first_child;
582         } while( (forker->first_child = forker->first_child->next) != start_child );
583
584         return NULL;
585 }
586
587
588 void del_prefork_child( prefork_simple* forker, pid_t pid ) { 
589
590         if( forker->first_child == NULL ) { return; }
591
592         (forker->current_num_children)--;
593         osrfLogDebug( OSRF_LOG_MARK, "Deleting Child: %d", pid );
594
595         prefork_child* start_child = forker->first_child; /* starting point */
596         prefork_child* cur_child        = start_child; /* current pointer */
597         prefork_child* prev_child       = start_child; /* the trailing pointer */
598
599         /* special case where there is only one in the list */
600         if( start_child == start_child->next ) {
601                 if( start_child->pid == pid ) {
602                         forker->first_child = NULL;
603
604                         close( start_child->read_data_fd );
605                         close( start_child->write_data_fd );
606                         close( start_child->read_status_fd );
607                         close( start_child->write_status_fd );
608
609                         prefork_child_free( start_child );
610                 }
611                 return;
612         }
613
614
615         /* special case where the first item in the list needs to be removed */
616         if( start_child->pid == pid ) { 
617
618                 /* find the last one so we can remove the start_child */
619                 do { 
620                         prev_child = cur_child;
621                         cur_child = cur_child->next;
622                 }while( cur_child != start_child );
623
624                 /* now cur_child == start_child */
625                 prev_child->next = cur_child->next;
626                 forker->first_child = prev_child;
627
628                 close( cur_child->read_data_fd );
629                 close( cur_child->write_data_fd );
630                 close( cur_child->read_status_fd );
631                 close( cur_child->write_status_fd );
632
633                 prefork_child_free( cur_child );
634                 return;
635         } 
636
637         do {
638                 prev_child = cur_child;
639                 cur_child = cur_child->next;
640
641                 if( cur_child->pid == pid ) {
642                         prev_child->next = cur_child->next;
643
644                         close( cur_child->read_data_fd );
645                         close( cur_child->write_data_fd );
646                         close( cur_child->read_status_fd );
647                         close( cur_child->write_status_fd );
648
649                         prefork_child_free( cur_child );
650                         return;
651                 }
652
653         } while(cur_child != start_child);
654 }
655
656
657
658
659 prefork_child* prefork_child_init( 
660         int max_requests, int read_data_fd, int write_data_fd, 
661         int read_status_fd, int write_status_fd ) {
662
663         prefork_child* child = (prefork_child*) safe_malloc(sizeof(prefork_child));
664         child->max_requests             = max_requests;
665         child->read_data_fd             = read_data_fd;
666         child->write_data_fd            = write_data_fd;
667         child->read_status_fd   = read_status_fd;
668         child->write_status_fd  = write_status_fd;
669         child->available                        = 1;
670
671         return child;
672 }
673
674
675 int prefork_free( prefork_simple* prefork ) {
676         
677         while( prefork->first_child != NULL ) {
678                 osrfLogInfo( OSRF_LOG_MARK,  "Killing children and sleeping 1 to reap..." );
679                 kill( 0,        SIGKILL );
680                 sleep(1);
681         }
682
683         client_free(prefork->connection);
684         free(prefork->appname);
685         free( prefork );
686         return 1;
687 }
688
689 int prefork_child_free( prefork_child* child ) { 
690         free(child->appname);
691         close(child->read_data_fd);
692         close(child->write_status_fd);
693         free( child ); 
694         return 1;
695 }
696