Log redaction for sensitive input values, C side
[OpenSRF.git] / src / gateway / osrf_json_gateway.c
1 #include "apachetools.h"
2 #include "opensrf/osrf_app_session.h"
3 #include "opensrf/osrf_system.h"
4 #include "opensrf/osrfConfig.h"
5 #include <opensrf/osrf_json.h>
6 #include <opensrf/osrf_json_xml.h>
7 #include <opensrf/osrf_legacy_json.h>
8 #include <opensrf/string_array.h>
9 #include <sys/time.h>
10 #include <sys/resource.h>
11 #include <unistd.h>
12 #include <strings.h>
13
14
15 #define MODULE_NAME "osrf_json_gateway_module"
16 #define GATEWAY_CONFIG "OSRFGatewayConfig"
17 #define DEFAULT_LOCALE "OSRFDefaultLocale"
18 #define CONFIG_CONTEXT "gateway"
19 #define JSON_PROTOCOL "OSRFGatewayLegacyJSON"
20 #define GATEWAY_USE_LEGACY_JSON 0
21
22 typedef struct {
23         int legacyJSON;
24 } osrf_json_gateway_dir_config;
25
26
27 module AP_MODULE_DECLARE_DATA osrf_json_gateway_module;
28
29 char* osrf_json_default_locale = "en-US";
30 char* osrf_json_gateway_config_file = NULL;
31 int bootstrapped = 0;
32 int numserved = 0;
33
34 static const char* osrf_json_gateway_set_default_locale(cmd_parms *parms,
35                 void *config, const char *arg) {
36         if (arg)
37                 osrf_json_default_locale = (char*) arg;
38         return NULL;
39 }
40
41 static const char* osrf_json_gateway_set_config(cmd_parms *parms, void *config, const char *arg) {
42         osrf_json_gateway_config_file = (char*) arg;
43         return NULL;
44 }
45
46 static const char* osrf_json_gateway_set_json_proto(cmd_parms *parms, void *config, const char *arg) {
47         osrf_json_gateway_dir_config* cfg = (osrf_json_gateway_dir_config*) config;
48         cfg->legacyJSON = (!strcasecmp((char*) arg, "true")) ? 1 : 0;
49         return NULL;
50 }
51
52 /* tell apache about our commands */
53 static const command_rec osrf_json_gateway_cmds[] = {
54         AP_INIT_TAKE1( GATEWAY_CONFIG, osrf_json_gateway_set_config,
55                         NULL, RSRC_CONF, "osrf json gateway config file"),
56         AP_INIT_TAKE1( DEFAULT_LOCALE, osrf_json_gateway_set_default_locale,
57                         NULL, RSRC_CONF, "osrf json gateway default locale"),
58         AP_INIT_TAKE1( JSON_PROTOCOL, osrf_json_gateway_set_json_proto,
59                         NULL, ACCESS_CONF, "osrf json gateway config file"),
60         {NULL}
61 };
62
63
64 static void* osrf_json_gateway_create_dir_config( apr_pool_t* p, char* dir) {
65         osrf_json_gateway_dir_config* cfg = (osrf_json_gateway_dir_config*)
66                         apr_palloc(p, sizeof(osrf_json_gateway_dir_config));
67         cfg->legacyJSON = GATEWAY_USE_LEGACY_JSON;
68         return (void*) cfg;
69 }
70
71 static apr_status_t child_exit(void* data) {
72         osrfLogInfo(OSRF_LOG_MARK, "Disconnecting on child cleanup...");
73         osrf_system_shutdown();
74         return OK;
75 }
76
77 static void osrf_json_gateway_child_init(apr_pool_t *p, server_rec *s) {
78
79         char* cfg = osrf_json_gateway_config_file;
80         char buf[32];
81         int t = time(NULL);
82         snprintf(buf, sizeof(buf), "%d", t);
83
84         if( ! osrfSystemBootstrapClientResc( cfg, CONFIG_CONTEXT, buf ) ) {
85                 ap_log_error( APLOG_MARK, APLOG_ERR, 0, s,
86                         "Unable to Bootstrap OpenSRF Client with config %s..", cfg);
87                 return;
88         }
89
90         bootstrapped = 1;
91         osrfLogInfo(OSRF_LOG_MARK, "Bootstrapping gateway child for requests");
92
93         // when this pool is cleaned up, it means the child
94         // process is going away.  register some cleanup code
95         // XXX causes us to disconnect even for clone()'d process cleanup (as in mod_cgi)
96         //apr_pool_cleanup_register(p, NULL, child_exit, apr_pool_cleanup_null);
97 }
98
99 static int osrf_json_gateway_method_handler (request_rec *r) {
100
101         /* make sure we're needed first thing*/
102         if (strcmp(r->handler, MODULE_NAME )) return DECLINED;
103
104
105         osrf_json_gateway_dir_config* dir_conf =
106                 ap_get_module_config(r->per_dir_config, &osrf_json_gateway_module);
107
108
109         /* provide 2 different JSON parsers and serializers to support legacy JSON */
110         jsonObject* (*parseJSONFunc) (const char*) = legacy_jsonParseString;
111         char* (*jsonToStringFunc) (const jsonObject*) = legacy_jsonObjectToJSON;
112
113         if(dir_conf->legacyJSON) {
114                 ap_log_rerror( APLOG_MARK, APLOG_DEBUG, 0, r, "Using legacy JSON");
115
116         } else {
117                 parseJSONFunc = jsonParse;
118                 jsonToStringFunc = jsonObjectToJSON;
119         }
120
121
122         osrfLogDebug(OSRF_LOG_MARK, "osrf gateway: entered request handler");
123
124         /* verify we are connected */
125         if( !bootstrapped || !osrfSystemGetTransportClient()) {
126                 ap_log_rerror( APLOG_MARK, APLOG_ERR, 0, r, "Cannot process request "
127                                 "because the OpenSRF JSON gateway has not been bootstrapped...");
128                 usleep( 100000 ); /* 100 milliseconds */
129                 exit(1);
130         }
131
132         osrfLogSetAppname("osrf_json_gw");
133         osrfAppSessionSetIngress("gateway-v1");
134
135         char* osrf_locale   = NULL;
136         char* param_locale  = NULL;  /* locale for this call */
137         char* service       = NULL;  /* service to connect to */
138         char* method        = NULL;  /* method to perform */
139         char* format        = NULL;  /* method to perform */
140         char* a_l           = NULL;  /* request api level */
141         char* input_format  = NULL;  /* POST data format, defaults to 'format' */
142         int   isXML         = 0;
143         int   api_level     = 1;
144
145         r->allowed |= (AP_METHOD_BIT << M_GET);
146         r->allowed |= (AP_METHOD_BIT << M_POST);
147
148         osrfLogDebug(OSRF_LOG_MARK, "osrf gateway: parsing URL params");
149         osrfStringArray* mparams = NULL;
150         osrfStringArray* params  = apacheParseParms(r); /* free me */
151         param_locale             = apacheGetFirstParamValue( params, "locale" );
152         service                  = apacheGetFirstParamValue( params, "service" );
153         method                   = apacheGetFirstParamValue( params, "method" );
154         format                   = apacheGetFirstParamValue( params, "format" );
155         input_format             = apacheGetFirstParamValue( params, "input_format" );
156         a_l                      = apacheGetFirstParamValue( params, "api_level" );
157         mparams                  = apacheGetParamValues( params, "param" ); /* free me */
158
159         if(format == NULL)
160                 format = strdup( "json" );
161         if(input_format == NULL)
162                 input_format = strdup( format );
163
164         /* set the user defined timeout value */
165         int timeout = 60;
166         char* tout = apacheGetFirstParamValue( params, "timeout" ); /* request timeout in seconds */
167         if( tout ) {
168                 timeout = atoi(tout);
169                 osrfLogDebug(OSRF_LOG_MARK, "Client supplied timeout of %d", timeout);
170                 free( tout );
171         }
172
173         if (a_l) {
174                 api_level = atoi(a_l);
175                 free( a_l );
176         }
177
178         if (!strcasecmp(format, "xml")) {
179                 isXML = 1;
180                 ap_set_content_type(r, "application/xml");
181         } else {
182                 ap_set_content_type(r, "text/plain");
183         }
184
185         free( format );
186         int ret = OK;
187
188         /* ----------------------------------------------------------------- */
189         /* Grab the requested locale using the Accept-Language header*/
190
191
192         if ( !param_locale ) {
193                 if ( apr_table_get(r->headers_in, "X-OpenSRF-Language") ) {
194                         param_locale = strdup( apr_table_get(r->headers_in, "X-OpenSRF-Language") );
195                 } else if ( apr_table_get(r->headers_in, "Accept-Language") ) {
196                         param_locale = strdup( apr_table_get(r->headers_in, "Accept-Language") );
197                 }
198         }
199
200
201         if (param_locale) {
202                 growing_buffer* osrf_locale_buf = buffer_init(16);
203                 if (index(param_locale, ',')) {
204                         int ind = index(param_locale, ',') - param_locale;
205                         int i;
206                         for ( i = 0; i < ind && i < 128; i++ )
207                                 buffer_add_char( osrf_locale_buf, param_locale[i] );
208                 } else {
209                         buffer_add( osrf_locale_buf, param_locale );
210                 }
211
212                 free(param_locale);
213                 osrf_locale = buffer_release( osrf_locale_buf );
214         } else {
215                 osrf_locale = strdup( osrf_json_default_locale );
216         }
217         /* ----------------------------------------------------------------- */
218
219
220         if(!(service && method)) {
221
222                 osrfLogError(OSRF_LOG_MARK,
223                         "Service [%s] not found or not allowed", service);
224                 ret = HTTP_NOT_FOUND;
225
226         } else {
227
228                 /* This will log all heaers to the apache error log
229                 const apr_array_header_t* arr = apr_table_elts(r->headers_in);
230                 const void* ptr;
231
232                 while( (ptr = apr_array_pop(arr)) ) {
233                         apr_table_entry_t* e = (apr_table_entry_t*) ptr;
234                         fprintf(stderr, "Table entry: %s : %s\n", e->key, e->val );
235                 }
236                 fflush(stderr);
237                 */
238
239                 osrfAppSession* session = osrfAppSessionClientInit(service);
240                 osrf_app_session_set_locale(session, osrf_locale);
241
242                 double starttime = get_timestamp_millis();
243                 int req_id = -1;
244
245                 if(!strcasecmp(input_format, "json")) {
246                         jsonObject * arr = jsonNewObject(NULL);
247
248                         const char* str;
249                         int i = 0;
250
251                         while( (str = osrfStringArrayGetString(mparams, i++)) )
252                                 jsonObjectPush(arr, parseJSONFunc(str));
253
254                         req_id = osrfAppSessionSendRequest( session, arr, method, api_level );
255                         jsonObjectFree(arr);
256                 } else {
257
258                         /**
259                         * If we receive XML method params, convert each param to a JSON object
260                         * and pass the array of JSON object params to the method */
261                         if(!strcasecmp(input_format, "xml")) {
262                                 jsonObject* jsonParams = jsonNewObject(NULL);
263
264                                 const char* str;
265                                 int i = 0;
266                                 while( (str = osrfStringArrayGetString(mparams, i++)) ) {
267                                         jsonObjectPush(jsonParams, jsonXMLToJSONObject(str));
268                                 }
269
270                                 req_id = osrfAppSessionSendRequest( session, jsonParams, method, api_level );
271                                 jsonObjectFree(jsonParams);
272                         }
273                 }
274
275
276                 if( req_id == -1 ) {
277                         osrfLogError(OSRF_LOG_MARK, "I am unable to communicate with opensrf..going away...");
278                         osrfAppSessionFree(session);
279                         /* we don't want to spawn an intense re-forking storm
280                          * if there is no jabber server.. so give it some time before we die */
281                         usleep( 100000 ); /* 100 milliseconds */
282                         exit(1);
283                 }
284
285
286                 /* ----------------------------------------------------------------- */
287                 /* log all requests to the activity log */
288                 const char* authtoken = apr_table_get(r->headers_in, "X-OILS-Authtoken");
289                 if(!authtoken) authtoken = "";
290                 growing_buffer* act = buffer_init(128);
291                 buffer_fadd(act, "[%s] [%s] [%s] %s %s", r->connection->remote_ip,
292                         authtoken, osrf_locale, service, method );
293
294                 const char* str; int i = 0;
295                 int redact_params = 0;
296                 while( (str = osrfStringArrayGetString(log_protect_arr, i++)) ) {
297                         //osrfLogInternal(OSRF_LOG_MARK, "Checking for log protection [%s]", str);
298                         if(!strncmp(method, str, strlen(str))) {
299                                 redact_params = 1;
300                                 break;
301                         }
302                 }
303                 if(redact_params) {
304                         OSRF_BUFFER_ADD(act, " **PARAMS REDACTED**");
305                 } else {
306                         i = 0;
307                         while( (str = osrfStringArrayGetString(mparams, i++)) ) {
308                                 if( i == 1 ) {
309                                         OSRF_BUFFER_ADD(act, " ");
310                                         OSRF_BUFFER_ADD(act, str);
311                                 } else {
312                                         OSRF_BUFFER_ADD(act, ", ");
313                                         OSRF_BUFFER_ADD(act, str);
314                                 }
315                         }
316                 }
317
318                 osrfLogActivity( OSRF_LOG_MARK, "%s", act->buf );
319                 buffer_free(act);
320                 /* ----------------------------------------------------------------- */
321
322
323                 osrfMessage* omsg = NULL;
324
325                 int statuscode = 200;
326
327                 /* kick off the object */
328                 if (isXML)
329                         ap_rputs( "<response xmlns=\"http://opensrf.org/-/namespaces/gateway/v1\"><payload>",
330                                 r );
331                 else
332                         ap_rputs("{\"payload\":[", r);
333
334                 int morethan1       = 0;
335                 char* statusname    = NULL;
336                 char* statustext    = NULL;
337                 char* output        = NULL;
338
339                 while((omsg = osrfAppSessionRequestRecv( session, req_id, timeout ))) {
340
341                         statuscode = omsg->status_code;
342                         const jsonObject* res;
343
344                         if( ( res = osrfMessageGetResult(omsg)) ) {
345
346                                 if (isXML) {
347                                         output = jsonObjectToXML( res );
348                                 } else {
349                                         output = jsonToStringFunc( res );
350                                         if( morethan1 ) ap_rputs(",", r); /* comma between JSON array items */
351                                 }
352                                 ap_rputs(output, r);
353                                 free(output);
354                                 morethan1 = 1;
355
356                         } else {
357
358                                 if( statuscode > 299 ) { /* the request returned a low level error */
359                                         statusname = omsg->status_name ? strdup(omsg->status_name)
360                                                 : strdup("Unknown Error");
361                                         statustext = omsg->status_text ? strdup(omsg->status_text) 
362                                                 : strdup("No Error Message");
363                                         osrfLogError( OSRF_LOG_MARK,  "Gateway received error: %s", statustext );
364                                 }
365                         }
366
367                         osrfMessageFree(omsg);
368                         if(statusname) break;
369                 }
370
371                 double duration = get_timestamp_millis() - starttime;
372                 osrfLogDebug(OSRF_LOG_MARK, "gateway request took %f seconds", duration);
373
374
375                 if (isXML)
376                         ap_rputs("</payload>", r);
377                 else
378                         ap_rputs("]",r); /* finish off the payload array */
379
380                 if(statusname) {
381
382                         /* add a debug field if the request died */
383                         ap_log_rerror( APLOG_MARK, APLOG_INFO, 0, r,
384                                         "OpenSRF JSON Request returned error: %s -> %s", statusname, statustext );
385                         int l = strlen(statusname) + strlen(statustext) + 32;
386                         char buf[l];
387
388                         if (isXML)
389                                 snprintf( buf, sizeof(buf), "<debug>\"%s : %s\"</debug>", statusname, statustext );
390
391                         else {
392                                 char bb[l];
393                                 snprintf(bb, sizeof(bb),  "%s : %s", statusname, statustext);
394                                 jsonObject* tmp = jsonNewObject(bb);
395                                 char* j = jsonToStringFunc(tmp);
396                                 snprintf( buf, sizeof(buf), ",\"debug\": %s", j);
397                                 free(j);
398                                 jsonObjectFree(tmp);
399                         }
400
401                         ap_rputs(buf, r);
402
403                         free(statusname);
404                         free(statustext);
405                 }
406
407                 /* insert the status code */
408                 char buf[32];
409
410                 if (isXML)
411                         snprintf(buf, sizeof(buf), "<status>%d</status>", statuscode );
412                 else
413                         snprintf(buf, sizeof(buf), ",\"status\":%d", statuscode );
414
415                 ap_rputs( buf, r );
416
417                 if (isXML)
418                         ap_rputs("</response>", r);
419                 else
420                         ap_rputs( "}", r ); /* finish off the object */
421
422                 osrfAppSessionFree(session);
423         }
424
425         osrfLogInfo(OSRF_LOG_MARK, "Completed processing service=%s, method=%s", service, method);
426         osrfStringArrayFree(params);
427         osrfStringArrayFree(mparams);
428         free( osrf_locale );
429         free( input_format );
430         free( method );
431         free( service );
432
433         osrfLogDebug(OSRF_LOG_MARK, "Gateway served %d requests", ++numserved);
434         osrfLogClearXid();
435
436         return ret;
437 }
438
439
440
441 static void osrf_json_gateway_register_hooks (apr_pool_t *p) {
442         ap_hook_handler(osrf_json_gateway_method_handler, NULL, NULL, APR_HOOK_MIDDLE);
443         ap_hook_child_init(osrf_json_gateway_child_init,NULL,NULL,APR_HOOK_MIDDLE);
444 }
445
446
447 module AP_MODULE_DECLARE_DATA osrf_json_gateway_module = {
448         STANDARD20_MODULE_STUFF,
449         osrf_json_gateway_create_dir_config,
450         NULL,
451         NULL,
452         NULL,
453         osrf_json_gateway_cmds,
454         osrf_json_gateway_register_hooks,
455 };