From 64f5284570b94856aad1ed8ad83afe71cb22cd66 Mon Sep 17 00:00:00 2001 From: scottmk Date: Sat, 1 May 2010 18:11:47 +0000 Subject: [PATCH] Moved the datatype lookups from the drones to the listener process. The cstore, rstore, and pcrud servers do dummy SELECT queries of all the non-virtual classes in the IDL, in order to get datatypes for all the non-virtual fields. These lookups are time-consuming. On my laptop they may take several seconds. On a system with various components running on separate boxes, they may take longer due to the additional network latency. Formerly these lookups were done by each drone process when it initialized itself, resulting in an occasional hiccup of additional response latency whenever the listener spawned another drone. Now the parent process does the lookup once, before it spawns any drones. The drones inherit the resulting datatypes via the normal fork operation, and never have to look up the datatypes themselves. In order for this new arrangement to work, the oilsExtendIDL function (which does the lookups) needs to receive a database connection as a parameter, since the parent's connection is different from those of the drones. M Open-ILS/include/openils/oils_sql.h M Open-ILS/src/c-apps/oils_pcrud.c M Open-ILS/src/c-apps/oils_rstore.c M Open-ILS/src/c-apps/oils_sql.c M Open-ILS/src/c-apps/oils_cstore.c git-svn-id: svn://svn.open-ils.org/ILS/trunk@16370 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/include/openils/oils_sql.h | 2 +- Open-ILS/src/c-apps/oils_cstore.c | 33 +++++++++++++++++------------ Open-ILS/src/c-apps/oils_pcrud.c | 30 +++++++++++++++----------- Open-ILS/src/c-apps/oils_rstore.c | 30 +++++++++++++++----------- Open-ILS/src/c-apps/oils_sql.c | 18 +++++++++++----- 5 files changed, 70 insertions(+), 43 deletions(-) diff --git a/Open-ILS/include/openils/oils_sql.h b/Open-ILS/include/openils/oils_sql.h index 121d87dfea..40c4cc0a00 100644 --- a/Open-ILS/include/openils/oils_sql.h +++ b/Open-ILS/include/openils/oils_sql.h @@ -29,7 +29,7 @@ extern "C" { dbi_conn oilsConnectDB( const char* mod_name ); void oilsSetSQLOptions( const char* module_name, int do_pcrud, int flesh_depth ); void oilsSetDBConnection( dbi_conn conn ); -int oilsExtendIDL( void ); +int oilsExtendIDL( dbi_conn handle ); int str_is_true( const char* str ); char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags ); diff --git a/Open-ILS/src/c-apps/oils_cstore.c b/Open-ILS/src/c-apps/oils_cstore.c index a22f306576..a2aa3fac73 100644 --- a/Open-ILS/src/c-apps/oils_cstore.c +++ b/Open-ILS/src/c-apps/oils_cstore.c @@ -25,7 +25,7 @@ static const char modulename[] = "open-ils.cstore"; This function is called when the server drone is about to terminate. */ -void osrfAppChildExit() { +void osrfAppChildExit( void ) { osrfLogDebug(OSRF_LOG_MARK, "Child is exiting, disconnecting from database..."); int same = 0; @@ -86,14 +86,27 @@ void osrfAppChildExit() { This function is called when the registering the application, and is executed by the listener before spawning the drones. */ -int osrfAppInitialize() { +int osrfAppInitialize( void ) { osrfLogInfo(OSRF_LOG_MARK, "Initializing the CStore Server..."); osrfLogInfo(OSRF_LOG_MARK, "Finding XML file..."); + // Load the IDL into memory if ( !oilsIDLInit( osrf_settings_host_value( "/IDL" ))) return 1; /* return non-zero to indicate error */ + // Open the database temporarily. Look up the datatypes of all + // the non-virtual fields and record them with the IDL data. + dbi_conn handle = oilsConnectDB( modulename ); + if( !handle ) + return -1; + else if( oilsExtendIDL( handle )) { + osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); + return -1; + } + dbi_conn_close( handle ); + + // Get the maximum flesh depth from the settings char* md = osrf_settings_host_value( "/apps/%s/app_settings/max_query_recursion", modulename ); int max_flesh_depth = 100; @@ -106,6 +119,7 @@ int osrfAppInitialize() { oilsSetSQLOptions( modulename, enforce_pcrud, max_flesh_depth ); + // Now register all the methods growing_buffer* method_name = buffer_init(64); // Generic search thingy @@ -114,7 +128,7 @@ int osrfAppInitialize() { osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR( method_name ), "doJSONSearch", "", 1, OSRF_METHOD_STREAMING ); - // first we register all the transaction and savepoint methods + // Next we register all the transaction and savepoint methods buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".transaction.begin"); @@ -267,12 +281,11 @@ int osrfAppInitialize() { @brief Initialize a server drone. @return Zero if successful, -1 if not. - Connect to the database. For each non-virtual class in the IDL, execute a dummy "SELECT * " - query to get the datatype of each column. Record the datatypes in the loaded IDL. + Connect to the database. This function is called by a server drone shortly after it is spawned by the listener. */ -int osrfAppChildInit() { +int osrfAppChildInit( void ) { writehandle = oilsConnectDB( modulename ); if( !writehandle ) @@ -280,13 +293,7 @@ int osrfAppChildInit() { oilsSetDBConnection( writehandle ); - // Add datatypes from database to the fields in the IDL - if( oilsExtendIDL() ) { - osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); - return -1; - } - else - return 0; + return 0; } /** diff --git a/Open-ILS/src/c-apps/oils_pcrud.c b/Open-ILS/src/c-apps/oils_pcrud.c index 8dbd48a499..b95bdae1b9 100644 --- a/Open-ILS/src/c-apps/oils_pcrud.c +++ b/Open-ILS/src/c-apps/oils_pcrud.c @@ -28,7 +28,7 @@ static const char modulename[] = "open-ils.pcrud"; This function is called when the server drone is about to terminate. */ -void osrfAppChildExit() { +void osrfAppChildExit( void ) { osrfLogDebug(OSRF_LOG_MARK, "Child is exiting, disconnecting from database..."); int same = 0; @@ -88,7 +88,7 @@ void osrfAppChildExit() { This function is called when the registering the application, and is executed by the listener before spawning the drones. */ -int osrfAppInitialize() { +int osrfAppInitialize( void ) { osrfLogInfo(OSRF_LOG_MARK, "Initializing the PCRUD Server..."); osrfLogInfo(OSRF_LOG_MARK, "Finding XML file..."); @@ -96,6 +96,18 @@ int osrfAppInitialize() { if ( !oilsIDLInit( osrf_settings_host_value( "/IDL" ))) return 1; /* return non-zero to indicate error */ + // Open the database temporarily. Look up the datatypes of all + // the non-virtual fields and record them with the IDL data. + dbi_conn handle = oilsConnectDB( modulename ); + if( !handle ) + return -1; + else if( oilsExtendIDL( handle )) { + osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); + return -1; + } + dbi_conn_close( handle ); + + // Get the maximum flesh depth from the settings char* md = osrf_settings_host_value( "/apps/%s/app_settings/max_query_recursion", modulename ); int max_flesh_depth = 100; @@ -108,6 +120,7 @@ int osrfAppInitialize() { oilsSetSQLOptions( modulename, enforce_pcrud, max_flesh_depth ); + // Now register all the methods growing_buffer* method_name = buffer_init(64); // first we register all the transaction and savepoint methods @@ -267,12 +280,11 @@ int osrfAppInitialize() { @brief Initialize a server drone. @return Zero if successful, -1 if not. - Connect to the database. For each non-virtual class in the IDL, execute a dummy "SELECT * " - query to get the datatype of each column. Record the datatypes in the loaded IDL. + Connect to the database. This function is called by a server drone shortly after it is spawned by the listener. */ -int osrfAppChildInit() { +int osrfAppChildInit( void ) { writehandle = oilsConnectDB( modulename ); if( !writehandle ) @@ -280,13 +292,7 @@ int osrfAppChildInit() { oilsSetDBConnection( writehandle ); - // Add datatypes from database to the fields in the IDL - if( oilsExtendIDL() ) { - osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); - return -1; - } - else - return 0; + return 0; } /** diff --git a/Open-ILS/src/c-apps/oils_rstore.c b/Open-ILS/src/c-apps/oils_rstore.c index 50e98e9595..ae76a7608c 100644 --- a/Open-ILS/src/c-apps/oils_rstore.c +++ b/Open-ILS/src/c-apps/oils_rstore.c @@ -25,7 +25,7 @@ static const char modulename[] = "open-ils.reporter-store"; This function is called when the server drone is about to terminate. */ -void osrfAppChildExit() { +void osrfAppChildExit( void ) { osrfLogDebug(OSRF_LOG_MARK, "Child is exiting, disconnecting from database..."); int same = 0; @@ -86,7 +86,7 @@ void osrfAppChildExit() { This function is called when the registering the application, and is executed by the listener before spawning the drones. */ -int osrfAppInitialize() { +int osrfAppInitialize( void ) { osrfLogInfo(OSRF_LOG_MARK, "Initializing the RStore Server..."); osrfLogInfo(OSRF_LOG_MARK, "Finding XML file..."); @@ -94,6 +94,18 @@ int osrfAppInitialize() { if ( !oilsIDLInit( osrf_settings_host_value( "/IDL" ))) return 1; /* return non-zero to indicate error */ + // Open the database temporarily. Look up the datatypes of all + // the non-virtual fields and record them with the IDL data. + dbi_conn handle = oilsConnectDB( modulename ); + if( !handle ) + return -1; + else if( oilsExtendIDL( handle )) { + osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); + return -1; + } + dbi_conn_close( handle ); + + // Get the maximum flesh depth from the settings char* md = osrf_settings_host_value( "/apps/%s/app_settings/max_query_recursion", modulename ); int max_flesh_depth = 100; @@ -106,6 +118,7 @@ int osrfAppInitialize() { oilsSetSQLOptions( modulename, enforce_pcrud, max_flesh_depth ); + // Now register all the methods growing_buffer* method_name = buffer_init(64); // Generic search thingy @@ -267,12 +280,11 @@ int osrfAppInitialize() { @brief Initialize a server drone. @return Zero if successful, -1 if not. - Connect to the database. For each non-virtual class in the IDL, execute a dummy "SELECT * " - query to get the datatype of each column. Record the datatypes in the loaded IDL. + Connect to the database. This function is called by a server drone shortly after it is spawned by the listener. */ -int osrfAppChildInit() { +int osrfAppChildInit( void ) { writehandle = oilsConnectDB( modulename ); if( !writehandle ) @@ -280,13 +292,7 @@ int osrfAppChildInit() { oilsSetDBConnection( writehandle ); - // Add datatypes from database to the fields in the IDL - if( oilsExtendIDL() ) { - osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); - return -1; - } - else - return 0; + return 0; } /** diff --git a/Open-ILS/src/c-apps/oils_sql.c b/Open-ILS/src/c-apps/oils_sql.c index d94846f2f5..40a79eb4f4 100644 --- a/Open-ILS/src/c-apps/oils_sql.c +++ b/Open-ILS/src/c-apps/oils_sql.c @@ -276,18 +276,18 @@ char* oilsGetRelation( osrfHash* classdef ) { /** @brief Add datatypes from the database to the fields in the IDL. + @param handle Handle for a database connection @return Zero if successful, or 1 upon error. For each relevant class in the IDL: ask the database for the datatype of every field. In particular, determine which fields are text fields and which fields are numeric fields, so that we know whether to enclose their values in quotes. - - At this writing this function does not detect any errors, so it always returns zero. */ -int oilsExtendIDL( void ) { +int oilsExtendIDL( dbi_conn handle ) { osrfHashIterator* class_itr = osrfNewHashIterator( oilsIDL() ); osrfHash* class = NULL; growing_buffer* query_buf = buffer_init( 64 ); + int results_found = 0; // boolean // For each class in the IDL... while( (class = osrfHashIteratorNext( class_itr ) ) ) { @@ -312,9 +312,10 @@ int oilsExtendIDL( void ) { osrfLogDebug( OSRF_LOG_MARK, "%s Investigatory SQL = %s", modulename, OSRF_BUFFER_C_STR( query_buf ) ); - dbi_result result = dbi_conn_query( writehandle, OSRF_BUFFER_C_STR( query_buf ) ); + dbi_result result = dbi_conn_query( handle, OSRF_BUFFER_C_STR( query_buf ) ); if( result ) { + results_found = 1; int columnIndex = 1; const char* columnName; while( (columnName = dbi_result_get_field_name(result, columnIndex)) ) { @@ -391,7 +392,14 @@ int oilsExtendIDL( void ) { buffer_free( query_buf ); osrfHashIteratorFree( class_itr ); child_initialized = 1; - return 0; + + if( !results_found ) { + osrfLogError( OSRF_LOG_MARK, + "No results found for any class -- bad database connection?" ); + return 1; + } + else + return 0; } /** -- 2.43.2