Open-ILS/src/c-apps/oils_sql.c

   1 /**
   2         @file oils_sql.c
   3         @brief Utility routines for translating JSON into SQL.
   4 */
   5
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <time.h>
   9 #include <ctype.h>
  10 #include <dbi/dbi.h>
  11 #include "opensrf/utils.h"
  12 #include "opensrf/log.h"
  13 #include "opensrf/osrf_application.h"
  14 #include "openils/oils_utils.h"
  15 #include "openils/oils_sql.h"
  16
  17 // The next four macros are OR'd together as needed to form a set
  18 // of bitflags.  SUBCOMBO enables an extra pair of parentheses when
  19 // nesting one UNION, INTERSECT or EXCEPT inside another.
  20 // SUBSELECT tells us we're in a subquery, so don't add the
  21 // terminal semicolon yet.
  22 #define SUBCOMBO    8
  23 #define SUBSELECT   4
  24 #define DISABLE_I18N    2
  25 #define SELECT_DISTINCT 1
  26
  27 #define AND_OP_JOIN     0
  28 #define OR_OP_JOIN      1
  29
  30 struct ClassInfoStruct;
  31 typedef struct ClassInfoStruct ClassInfo;
  32
  33 #define ALIAS_STORE_SIZE 16
  34 #define CLASS_NAME_STORE_SIZE 16
  35
  36 struct ClassInfoStruct {
  37         char* alias;
  38         char* class_name;
  39         char* source_def;
  40         osrfHash* class_def;      // Points into IDL
  41         osrfHash* fields;         // Points into IDL
  42         osrfHash* links;          // Points into IDL
  43
  44         // The remaining members are private and internal.  Client code should not
  45         // access them directly.
  46
  47         ClassInfo* next;          // Supports linked list of joined classes
  48         int in_use;               // boolean
  49
  50         // We usually store the alias and class name in the following arrays, and
  51         // point the corresponding pointers at them.  When the string is too big
  52         // for the array (which will probably never happen in practice), we strdup it.
  53
  54         char alias_store[ ALIAS_STORE_SIZE + 1 ];
  55         char class_name_store[ CLASS_NAME_STORE_SIZE + 1 ];
  56 };
  57
  58 struct QueryFrameStruct;
  59 typedef struct QueryFrameStruct QueryFrame;
  60
  61 struct QueryFrameStruct {
  62         ClassInfo core;
  63         ClassInfo* join_list;  // linked list of classes joined to the core class
  64         QueryFrame* next;      // implements stack as linked list
  65         int in_use;            // boolean
  66 };
  67
  68 static int timeout_needs_resetting;
  69 static time_t time_next_reset;
  70
  71 static int verifyObjectClass ( osrfMethodContext*, const jsonObject* );
  72
  73 static void setXactId( osrfMethodContext* ctx );
  74 static inline const char* getXactId( osrfMethodContext* ctx );
  75 static inline void clearXactId( osrfMethodContext* ctx );
  76
  77 static jsonObject* doFieldmapperSearch ( osrfMethodContext* ctx, osrfHash* class_meta,
  78                 jsonObject* where_hash, jsonObject* query_hash, int* err );
  79 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result, osrfHash* );
  80 static jsonObject* oilsMakeJSONFromResult( dbi_result );
  81
  82 static char* searchSimplePredicate ( const char* op, const char* class_alias,
  83                                 osrfHash* field, const jsonObject* node );
  84 static char* searchFunctionPredicate ( const char*, osrfHash*, const jsonObject*, const char* );
  85 static char* searchFieldTransform ( const char*, osrfHash*, const jsonObject* );
  86 static char* searchFieldTransformPredicate ( const ClassInfo*, osrfHash*, const jsonObject*,
  87                 const char* );
  88 static char* searchBETWEENRange( osrfHash* field, const jsonObject* node );
  89 static char* searchBETWEENPredicate ( const char*, osrfHash*, const jsonObject* );
  90 static char* searchINList( osrfHash* field, jsonObject* node, const char* op, osrfMethodContext* ctx );
  91 static char* searchINPredicate ( const char*, osrfHash*,
  92                                                                  jsonObject*, const char*, osrfMethodContext* );
  93 static char* searchPredicate ( const ClassInfo*, osrfHash*, jsonObject*, osrfMethodContext* );
  94 static char* searchJOIN ( const jsonObject*, const ClassInfo* left_info );
  95 static char* searchWHERE ( const jsonObject* search_hash, const ClassInfo*, int, osrfMethodContext* );
  96 static char* buildSELECT( const jsonObject*, jsonObject* rest_of_query,
  97         osrfHash* meta, osrfMethodContext* ctx );
  98 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array );
  99
 100 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags );
 101
 102 char* SELECT ( osrfMethodContext*, jsonObject*, const jsonObject*, const jsonObject*,
 103         const jsonObject*, const jsonObject*, const jsonObject*, const jsonObject*, int );
 104
 105 static osrfStringArray* getPermLocationCache( osrfMethodContext*, const char* );
 106 static void setPermLocationCache( osrfMethodContext*, const char*, osrfStringArray* );
 107
 108 void userDataFree( void* );
 109 static void sessionDataFree( char*, void* );
 110 static void pcacheFree( char*, void* );
 111 static int obj_is_true( const jsonObject* obj );
 112 static const char* json_type( int code );
 113 static const char* get_primitive( osrfHash* field );
 114 static const char* get_datatype( osrfHash* field );
 115 static void pop_query_frame( void );
 116 static void push_query_frame( void );
 117 static int add_query_core( const char* alias, const char* class_name );
 118 static inline ClassInfo* search_alias( const char* target );
 119 static ClassInfo* search_all_alias( const char* target );
 120 static ClassInfo* add_joined_class( const char* alias, const char* classname );
 121 static void clear_query_stack( void );
 122
 123 static const jsonObject* verifyUserPCRUD( osrfMethodContext* );
 124 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext*, int );
 125 static int verifyObjectPCRUD( osrfMethodContext*, osrfHash*, const jsonObject*, int );
 126 static const char* org_tree_root( osrfMethodContext* ctx );
 127 static jsonObject* single_hash( const char* key, const char* value );
 128
 129 static int child_initialized = 0;   /* boolean */
 130
 131 static dbi_conn writehandle; /* our MASTER db connection */
 132 static dbi_conn dbhandle; /* our CURRENT db connection */
 133 //static osrfHash * readHandles;
 134
 135 // The following points to the top of a stack of QueryFrames.  It's a little
 136 // confusing because the top level of the query is at the bottom of the stack.
 137 static QueryFrame* curr_query = NULL;
 138
 139 static dbi_conn writehandle; /* our MASTER db connection */
 140 static dbi_conn dbhandle; /* our CURRENT db connection */
 141 //static osrfHash * readHandles;
 142
 143 static int max_flesh_depth = 100;
 144
 145 static int perm_at_threshold = 5;
 146 static int enforce_pcrud = 0;     // Boolean
 147 static char* modulename = NULL;
 148
 149 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id);
 150
 151 static char* _sanitize_tz_name( const char* tz );
 152 static char* _sanitize_savepoint_name( const char* sp );
 153
 154 /**
 155         @brief Connect to the database.
 156         @return A database connection if successful, or NULL if not.
 157 */
 158 dbi_conn oilsConnectDB( const char* mod_name ) {
 159
 160         osrfLogDebug( OSRF_LOG_MARK, "Attempting to initialize libdbi..." );
 161         if( dbi_initialize( NULL ) == -1 ) {
 162                 osrfLogError( OSRF_LOG_MARK, "Unable to initialize libdbi" );
 163                 return NULL;
 164         } else
 165                 osrfLogDebug( OSRF_LOG_MARK, "... libdbi initialized." );
 166
 167         char* driver = osrf_settings_host_value( "/apps/%s/app_settings/driver", mod_name );
 168         char* user   = osrf_settings_host_value( "/apps/%s/app_settings/database/user", mod_name );
 169         char* host   = osrf_settings_host_value( "/apps/%s/app_settings/database/host", mod_name );
 170         char* port   = osrf_settings_host_value( "/apps/%s/app_settings/database/port", mod_name );
 171         char* db     = osrf_settings_host_value( "/apps/%s/app_settings/database/db", mod_name );
 172         char* pw     = osrf_settings_host_value( "/apps/%s/app_settings/database/pw", mod_name );
 173         char* pg_app = osrf_settings_host_value( "/apps/%s/app_settings/database/application_name", mod_name );
 174
 175         osrfLogDebug( OSRF_LOG_MARK, "Attempting to load the database driver [%s]...", driver );
 176         dbi_conn handle = dbi_conn_new( driver );
 177
 178         if( !handle ) {
 179                 osrfLogError( OSRF_LOG_MARK, "Error loading database driver [%s]", driver );
 180                 return NULL;
 181         }
 182         osrfLogDebug( OSRF_LOG_MARK, "Database driver [%s] seems OK", driver );
 183
 184         osrfLogInfo(OSRF_LOG_MARK, "%s connecting to database.  host=%s, "
 185                 "port=%s, user=%s, db=%s", mod_name, host, port, user, db );
 186
 187         if( host )   dbi_conn_set_option( handle, "host", host );
 188         if( port )   dbi_conn_set_option_numeric( handle, "port", atoi( port ));
 189         if( user )   dbi_conn_set_option( handle, "username", user );
 190         if( pw )     dbi_conn_set_option( handle, "password", pw );
 191         if( db )     dbi_conn_set_option( handle, "dbname", db );
 192         if( pg_app ) dbi_conn_set_option( handle, "pgsql_application_name", pg_app );
 193
 194         free( user );
 195         free( host );
 196         free( port );
 197         free( db );
 198         free( pw );
 199         free( pg_app );
 200
 201         if( dbi_conn_connect( handle ) < 0 ) {
 202                 sleep( 1 );
 203                 if( dbi_conn_connect( handle ) < 0 ) {
 204                         const char* msg;
 205                         dbi_conn_error( handle, &msg );
 206                         osrfLogError( OSRF_LOG_MARK, "Error connecting to database: %s",
 207                                 msg ? msg : "(No description available)" );
 208                         return NULL;
 209                 }
 210         }
 211
 212         osrfLogInfo( OSRF_LOG_MARK, "%s successfully connected to the database", mod_name );
 213
 214         return handle;
 215 }
 216
 217 /**
 218         @brief Select some options.
 219         @param module_name: Name of the server.
 220         @param do_pcrud: Boolean.  True if we are to enforce PCRUD permissions.
 221
 222         This source file is used (at this writing) to implement three different servers:
 223         - open-ils.reporter-store
 224         - open-ils.pcrud
 225         - open-ils.cstore
 226
 227         These servers behave mostly the same, but they implement different combinations of
 228         methods, and open-ils.pcrud enforces a permissions scheme that the other two don't.
 229
 230         Here we use the server name in messages to identify which kind of server issued them.
 231         We use do_crud as a boolean to control whether or not to enforce the permissions scheme.
 232 */
 233 void oilsSetSQLOptions( const char* module_name, int do_pcrud, int flesh_depth ) {
 234         if( !module_name )
 235                 module_name = "open-ils.cstore";   // bulletproofing with a default
 236
 237         if( modulename )
 238                 free( modulename );
 239
 240         modulename = strdup( module_name );
 241         enforce_pcrud = do_pcrud;
 242         max_flesh_depth = flesh_depth;
 243 }
 244
 245 /**
 246         @brief Install a database connection.
 247         @param conn Pointer to a database connection.
 248
 249         In some contexts, @a conn may merely provide a driver so that we can process strings
 250         properly, without providing an open database connection.
 251 */
 252 void oilsSetDBConnection( dbi_conn conn ) {
 253         dbhandle = writehandle = conn;
 254 }
 255
 256 /**
 257         @brief Determine whether a database connection is alive.
 258         @param handle Handle for a database connection.
 259         @return 1 if the connection is alive, or zero if it isn't.
 260 */
 261 int oilsIsDBConnected( dbi_conn handle ) {
 262         // Do an innocuous SELECT.  If it succeeds, the database connection is still good.
 263         dbi_result result = dbi_conn_query( handle, "SELECT 1;" );
 264         if( result ) {
 265                 dbi_result_free( result );
 266                 return 1;
 267         } else {
 268                 // This is a terrible, horrible, no good, very bad kludge.
 269                 // Sometimes the SELECT 1 query fails, not because the database connection is dead,
 270                 // but because (due to a previous error) the database is ignoring all commands,
 271                 // even innocuous SELECTs, until the current transaction is rolled back.  The only
 272                 // known way to detect this condition via the dbi library is by looking at the error
 273                 // message.  This approach will break if the language or wording of the message ever
 274                 // changes.
 275                 // Note: the dbi_conn_ping function purports to determine whether the database
 276                 // connection is live, but at this writing this function is unreliable and useless.
 277                 static const char* ok_msg = "ERROR:  current transaction is aborted, commands "
 278                         "ignored until end of transaction block\n";
 279                 const char* msg;
 280                 dbi_conn_error( handle, &msg );
 281                 // Newer versions of dbi_conn_error return codes within the error msg.
 282                 // E.g. 3624914: ERROR:  current transaction is aborted, commands ignored until end of transaction block
 283                 // Substring test should work regardless.
 284                 const char* substr = strstr(msg, ok_msg);
 285                 if( substr == NULL ) {
 286                         osrfLogError( OSRF_LOG_MARK, "Database connection isn't working : %s", msg );
 287                         return 0;
 288                 } else
 289                         return 1;   // ignoring SELECT due to previous error; that's okay
 290         }
 291 }
 292
 293 /**
 294         @brief Get a table name, view name, or subquery for use in a FROM clause.
 295         @param class Pointer to the IDL class entry.
 296         @return A table name, a view name, or a subquery in parentheses.
 297
 298         In some cases the IDL defines a class, not with a table name or a view name, but with
 299         a SELECT statement, which may be used as a subquery.
 300 */
 301 char* oilsGetRelation( osrfHash* classdef ) {
 302
 303         char* source_def = NULL;
 304         const char* tabledef = osrfHashGet( classdef, "tablename" );
 305
 306         if( tabledef ) {
 307                 source_def = strdup( tabledef );   // Return the name of a table or view
 308         } else {
 309                 tabledef = osrfHashGet( classdef, "source_definition" );
 310                 if( tabledef ) {
 311                         // Return a subquery, enclosed in parentheses
 312                         source_def = safe_malloc( strlen( tabledef ) + 3 );
 313                         source_def[ 0 ] = '(';
 314                         strcpy( source_def + 1, tabledef );
 315                         strcat( source_def, ")" );
 316                 } else {
 317                         // Not found: return an error
 318                         const char* classname = osrfHashGet( classdef, "classname" );
 319                         if( !classname )
 320                                 classname = "???";
 321                         osrfLogError(
 322                                 OSRF_LOG_MARK,
 323                                 "%s ERROR No tablename or source_definition for class \"%s\"",
 324                                 modulename,
 325                                 classname
 326                         );
 327                 }
 328         }
 329
 330         return source_def;
 331 }
 332
 333 /**
 334         @brief Add datatypes from the database to the fields in the IDL.
 335         @param handle Handle for a database connection
 336         @return Zero if successful, or 1 upon error.
 337
 338         For each relevant class in the IDL: ask the database for the datatype of every field.
 339         In particular, determine which fields are text fields and which fields are numeric
 340         fields, so that we know whether to enclose their values in quotes.
 341 */
 342 int oilsExtendIDL( dbi_conn handle ) {
 343         osrfHashIterator* class_itr = osrfNewHashIterator( oilsIDL() );
 344         osrfHash* class = NULL;
 345         growing_buffer* query_buf = buffer_init( 64 );
 346         int results_found = 0;   // boolean
 347
 348         // For each class in the IDL...
 349         while( (class = osrfHashIteratorNext( class_itr ) ) ) {
 350                 const char* classname = osrfHashIteratorKey( class_itr );
 351                 osrfHash* fields = osrfHashGet( class, "fields" );
 352
 353                 // If the class is virtual, ignore it
 354                 if( str_is_true( osrfHashGet(class, "virtual") ) ) {
 355                         osrfLogDebug(OSRF_LOG_MARK, "Class %s is virtual, skipping", classname );
 356                         continue;
 357                 }
 358
 359                 char* tabledef = oilsGetRelation( class );
 360                 if( !tabledef )
 361                         continue;   // No such relation -- a query of it would be doomed to failure
 362
 363                 buffer_reset( query_buf );
 364                 buffer_fadd( query_buf, "SELECT * FROM %s AS x WHERE 1=0;", tabledef );
 365
 366                 free(tabledef );
 367
 368                 osrfLogDebug( OSRF_LOG_MARK, "%s Investigatory SQL = %s",
 369                                 modulename, OSRF_BUFFER_C_STR( query_buf ) );
 370
 371                 dbi_result result = dbi_conn_query( handle, OSRF_BUFFER_C_STR( query_buf ) );
 372                 if( result ) {
 373
 374                         results_found = 1;
 375                         int columnIndex = 1;
 376                         const char* columnName;
 377                         while( (columnName = dbi_result_get_field_name(result, columnIndex)) ) {
 378
 379                                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...",
 380                                                 columnName );
 381
 382                                 /* fetch the fieldmapper index */
 383                                 osrfHash* _f = osrfHashGet(fields, columnName);
 384                                 if( _f ) {
 385
 386                                         osrfLogDebug(OSRF_LOG_MARK, "Found [%s] in IDL hash...", columnName);
 387
 388                                         /* determine the field type and storage attributes */
 389
 390                                         switch( dbi_result_get_field_type_idx( result, columnIndex )) {
 391
 392                                                 case DBI_TYPE_INTEGER : {
 393
 394                                                         if( !osrfHashGet(_f, "primitive") )
 395                                                                 osrfHashSet(_f, "number", "primitive");
 396
 397                                                         int attr = dbi_result_get_field_attribs_idx( result, columnIndex );
 398                                                         if( attr & DBI_INTEGER_SIZE8 )
 399                                                                 osrfHashSet( _f, "INT8", "datatype" );
 400                                                         else
 401                                                                 osrfHashSet( _f, "INT", "datatype" );
 402                                                         break;
 403                                                 }
 404                                                 case DBI_TYPE_DECIMAL :
 405                                                         if( !osrfHashGet( _f, "primitive" ))
 406                                                                 osrfHashSet( _f, "number", "primitive" );
 407
 408                                                         osrfHashSet( _f, "NUMERIC", "datatype" );
 409                                                         break;
 410
 411                                                 case DBI_TYPE_STRING :
 412                                                         if( !osrfHashGet( _f, "primitive" ))
 413                                                                 osrfHashSet( _f, "string", "primitive" );
 414
 415                                                         osrfHashSet( _f,"TEXT", "datatype" );
 416                                                         break;
 417
 418                                                 case DBI_TYPE_DATETIME :
 419                                                         if( !osrfHashGet( _f, "primitive" ))
 420                                                                 osrfHashSet( _f, "string", "primitive" );
 421
 422                                                         osrfHashSet( _f, "TIMESTAMP", "datatype" );
 423                                                         break;
 424
 425                                                 case DBI_TYPE_BINARY :
 426                                                         if( !osrfHashGet( _f, "primitive" ))
 427                                                                 osrfHashSet( _f, "string", "primitive" );
 428
 429                                                         osrfHashSet( _f, "BYTEA", "datatype" );
 430                                         }
 431
 432                                         osrfLogDebug(
 433                                                 OSRF_LOG_MARK,
 434                                                 "Setting [%s] to primitive [%s] and datatype [%s]...",
 435                                                 columnName,
 436                                                 osrfHashGet( _f, "primitive" ),
 437                                                 osrfHashGet( _f, "datatype" )
 438                                         );
 439                                 }
 440                                 ++columnIndex;
 441                         } // end while loop for traversing columns of result
 442                         dbi_result_free( result  );
 443                 } else {
 444                         const char* msg;
 445                         int errnum = dbi_conn_error( handle, &msg );
 446                         osrfLogDebug( OSRF_LOG_MARK, "No data found for class [%s]: %d, %s", classname,
 447                                 errnum, msg ? msg : "(No description available)" );
 448                         // We don't check the database connection here.  It's routine to get failures at
 449                         // this point; we routinely try to query tables that don't exist, because they
 450                         // are defined in the IDL but not in the database.
 451                 }
 452         } // end for each class in IDL
 453
 454         buffer_free( query_buf );
 455         osrfHashIteratorFree( class_itr );
 456         child_initialized = 1;
 457
 458         if( !results_found ) {
 459                 osrfLogError( OSRF_LOG_MARK,
 460                         "No results found for any class -- bad database connection?" );
 461                 return 1;
 462         } else if( ! oilsIsDBConnected( handle )) {
 463                 osrfLogError( OSRF_LOG_MARK,
 464                         "Unable to extend IDL: database connection isn't working" );
 465                 return 1;
 466         }
 467         else
 468                 return 0;
 469 }
 470
 471 /**
 472         @brief Free an osrfHash that stores a transaction ID.
 473         @param blob A pointer to the osrfHash to be freed, cast to a void pointer.
 474
 475         This function is a callback, to be called by the application session when it ends.
 476         The application session stores the osrfHash via an opaque pointer.
 477
 478         If the osrfHash contains an entry for the key "xact_id", it means that an
 479         uncommitted transaction is pending.  Roll it back.
 480 */
 481 void userDataFree( void* blob ) {
 482         osrfHash* hash = (osrfHash*) blob;
 483         if( osrfHashGet( hash, "xact_id" ) && writehandle ) {
 484                 if( !dbi_conn_query( writehandle, "ROLLBACK;" )) {
 485                         const char* msg;
 486                         int errnum = dbi_conn_error( writehandle, &msg );
 487                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform rollback: %d %s",
 488                                 errnum, msg ? msg : "(No description available)" );
 489                 };
 490         }
 491         if( writehandle ) {
 492                 if( !dbi_conn_query( writehandle, "SELECT auditor.clear_audit_info();" ) ) {
 493                         const char* msg;
 494                         int errnum = dbi_conn_error( writehandle, &msg );
 495                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform audit info clearing: %d %s",
 496                                 errnum, msg ? msg : "(No description available)" );
 497                 }
 498         }
 499
 500         osrfHashFree( hash );
 501 }
 502
 503 /**
 504         @name Managing session data
 505         @brief Maintain data stored via the userData pointer of the application session.
 506
 507         Currently, session-level data is stored in an osrfHash.  Other arrangements are
 508         possible, and some would be more efficient.  The application session calls a
 509         callback function to free userData before terminating.
 510
 511         Currently, the only data we store at the session level is the transaction id.  By this
 512         means we can ensure that any pending transactions are rolled back before the application
 513         session terminates.
 514 */
 515 /*@{*/
 516
 517 /**
 518         @brief Free an item in the application session's userData.
 519         @param key The name of a key for an osrfHash.
 520         @param item An opaque pointer to the item associated with the key.
 521
 522         We store an osrfHash as userData with the application session, and arrange (by
 523         installing userDataFree() as a different callback) for the session to free that
 524         osrfHash before terminating.
 525
 526         This function is a callback for freeing items in the osrfHash.  Currently we store
 527         two things:
 528         - Transaction id of a pending transaction; a character string.  Key: "xact_id".
 529         - Authkey; a character string.  Key: "authkey".
 530         - User object from the authentication server; a jsonObject.  Key: "user_login".
 531
 532         If we ever store anything else in userData, we will need to revisit this function so
 533         that it will free whatever else needs freeing.
 534 */
 535 static void sessionDataFree( char* key, void* item ) {
 536         if( !strcmp( key, "xact_id" ) || !strcmp( key, "authkey" ) || !strncmp( key, "rs_size_", 8) )
 537                 free( item );
 538         else if( !strcmp( key, "user_login" ) )
 539                 jsonObjectFree( (jsonObject*) item );
 540         else if( !strcmp( key, "pcache" ) )
 541                 osrfHashFree( (osrfHash*) item );
 542 }
 543
 544 static void pcacheFree( char* key, void* item ) {
 545         osrfStringArrayFree( (osrfStringArray*) item );
 546 }
 547
 548 /**
 549         @brief Initialize session cache.
 550         @param ctx Pointer to the method context.
 551
 552         Create a cache for the session by making the session's userData member point
 553         to an osrfHash instance.
 554 */
 555 static osrfHash* initSessionCache( osrfMethodContext* ctx ) {
 556         ctx->session->userData = osrfNewHash();
 557         osrfHashSetCallback( (osrfHash*) ctx->session->userData, &sessionDataFree );
 558         ctx->session->userDataFree = &userDataFree;
 559         return ctx->session->userData;
 560 }
 561
 562 /**
 563         @brief Save a transaction id.
 564         @param ctx Pointer to the method context.
 565
 566         Save the session_id of the current application session as a transaction id.
 567 */
 568 static void setXactId( osrfMethodContext* ctx ) {
 569         if( ctx && ctx->session ) {
 570                 osrfAppSession* session = ctx->session;
 571
 572                 osrfHash* cache = session->userData;
 573
 574                 // If the session doesn't already have a hash, create one.  Make sure
 575                 // that the application session frees the hash when it terminates.
 576                 if( NULL == cache )
 577                         cache = initSessionCache( ctx );
 578
 579                 // Save the transaction id in the hash, with the key "xact_id"
 580                 osrfHashSet( cache, strdup( session->session_id ), "xact_id" );
 581         }
 582 }
 583
 584 /**
 585         @brief Get the transaction ID for the current transaction, if any.
 586         @param ctx Pointer to the method context.
 587         @return Pointer to the transaction ID.
 588
 589         The return value points to an internal buffer, and will become invalid upon issuing
 590         a commit or rollback.
 591 */
 592 static inline const char* getXactId( osrfMethodContext* ctx ) {
 593         if( ctx && ctx->session && ctx->session->userData )
 594                 return osrfHashGet( (osrfHash*) ctx->session->userData, "xact_id" );
 595         else
 596                 return NULL;
 597 }
 598
 599 /**
 600         @brief Clear the current transaction id.
 601         @param ctx Pointer to the method context.
 602 */
 603 static inline void clearXactId( osrfMethodContext* ctx ) {
 604         if( ctx && ctx->session && ctx->session->userData )
 605                 osrfHashRemove( ctx->session->userData, "xact_id" );
 606 }
 607 /*@}*/
 608
 609 /**
 610         @brief Stash the location for a particular perm in the sessionData cache
 611         @param ctx Pointer to the method context.
 612         @param perm Name of the permission we're looking at
 613         @param array StringArray of perm location ids
 614 */
 615 static void setPermLocationCache( osrfMethodContext* ctx, const char* perm, osrfStringArray* locations ) {
 616         if( ctx && ctx->session ) {
 617                 osrfAppSession* session = ctx->session;
 618
 619                 osrfHash* cache = session->userData;
 620
 621                 // If the session doesn't already have a hash, create one.  Make sure
 622                 // that the application session frees the hash when it terminates.
 623                 if( NULL == cache )
 624                         cache = initSessionCache( ctx );
 625
 626                 osrfHash* pcache = osrfHashGet(cache, "pcache");
 627
 628                 if( NULL == pcache ) {
 629                         pcache = osrfNewHash();
 630                         osrfHashSetCallback( pcache, &pcacheFree );
 631                         osrfHashSet( cache, pcache, "pcache" );
 632                 }
 633
 634                 if( perm && locations )
 635                         osrfHashSet( pcache, locations, strdup(perm) );
 636         }
 637 }
 638
 639 /**
 640         @brief Grab stashed location for a particular perm in the sessionData cache
 641         @param ctx Pointer to the method context.
 642         @param perm Name of the permission we're looking at
 643 */
 644 static osrfStringArray* getPermLocationCache( osrfMethodContext* ctx, const char* perm ) {
 645         if( ctx && ctx->session ) {
 646                 osrfAppSession* session = ctx->session;
 647                 osrfHash* cache = session->userData;
 648                 if( cache ) {
 649                         osrfHash* pcache = osrfHashGet(cache, "pcache");
 650                         if( pcache ) {
 651                                 return osrfHashGet( pcache, perm );
 652                         }
 653                 }
 654         }
 655
 656         return NULL;
 657 }
 658
 659 /**
 660         @brief Save the user's login in the userData for the current application session.
 661         @param ctx Pointer to the method context.
 662         @param user_login Pointer to the user login object to be cached (we cache the original,
 663         not a copy of it).
 664
 665         If @a user_login is NULL, remove the user login if one is already cached.
 666 */
 667 static void setUserLogin( osrfMethodContext* ctx, jsonObject* user_login ) {
 668         if( ctx && ctx->session ) {
 669                 osrfAppSession* session = ctx->session;
 670
 671                 osrfHash* cache = session->userData;
 672
 673                 // If the session doesn't already have a hash, create one.  Make sure
 674                 // that the application session frees the hash when it terminates.
 675                 if( NULL == cache )
 676                         cache = initSessionCache( ctx );
 677
 678                 if( user_login )
 679                         osrfHashSet( cache, user_login, "user_login" );
 680                 else
 681                         osrfHashRemove( cache, "user_login" );
 682         }
 683 }
 684
 685 /**
 686         @brief Get the user login object for the current application session, if any.
 687         @param ctx Pointer to the method context.
 688         @return Pointer to the user login object if found; otherwise NULL.
 689
 690         The user login object was returned from the authentication server, and then cached so
 691         we don't have to call the authentication server again for the same user.
 692 */
 693 static const jsonObject* getUserLogin( osrfMethodContext* ctx ) {
 694         if( ctx && ctx->session && ctx->session->userData )
 695                 return osrfHashGet( (osrfHash*) ctx->session->userData, "user_login" );
 696         else
 697                 return NULL;
 698 }
 699
 700 /**
 701         @brief Save a copy of an authkey in the userData of the current application session.
 702         @param ctx Pointer to the method context.
 703         @param authkey The authkey to be saved.
 704
 705         If @a authkey is NULL, remove the authkey if one is already cached.
 706 */
 707 static void setAuthkey( osrfMethodContext* ctx, const char* authkey ) {
 708         if( ctx && ctx->session && authkey ) {
 709                 osrfAppSession* session = ctx->session;
 710                 osrfHash* cache = session->userData;
 711
 712                 // If the session doesn't already have a hash, create one.  Make sure
 713                 // that the application session frees the hash when it terminates.
 714                 if( NULL == cache )
 715                         cache = initSessionCache( ctx );
 716
 717                 // Save the transaction id in the hash, with the key "xact_id"
 718                 if( authkey && *authkey )
 719                         osrfHashSet( cache, strdup( authkey ), "authkey" );
 720                 else
 721                         osrfHashRemove( cache, "authkey" );
 722         }
 723 }
 724
 725 /**
 726         @brief Reset the login timeout.
 727         @param authkey The authentication key for the current login session.
 728         @param now The current time.
 729         @return Zero if successful, or 1 if not.
 730
 731         Tell the authentication server to reset the timeout so that the login session won't
 732         expire for a while longer.
 733
 734         We could dispense with the @a now parameter by calling time().  But we just called
 735         time() in order to decide whether to reset the timeout, so we might as well reuse
 736         the result instead of calling time() again.
 737 */
 738 static int reset_timeout( const char* authkey, time_t now ) {
 739         jsonObject* auth_object = jsonNewObject( authkey );
 740
 741         // Ask the authentication server to reset the timeout.  It returns an event
 742         // indicating success or failure.
 743         jsonObject* result = oilsUtilsQuickReq( "open-ils.auth",
 744                 "open-ils.auth.session.reset_timeout", auth_object );
 745         jsonObjectFree( auth_object );
 746
 747         if( !result || result->type != JSON_HASH ) {
 748                 osrfLogError( OSRF_LOG_MARK,
 749                          "Unexpected object type receieved from open-ils.auth.session.reset_timeout" );
 750                 jsonObjectFree( result );
 751                 return 1;       // Not the right sort of object returned
 752         }
 753
 754         const jsonObject* ilsevent = jsonObjectGetKeyConst( result, "ilsevent" );
 755         if( !ilsevent || ilsevent->type != JSON_NUMBER ) {
 756                 osrfLogError( OSRF_LOG_MARK, "ilsevent is absent or malformed" );
 757                 jsonObjectFree( result );
 758                 return 1;    // Return code from method not available
 759         }
 760
 761         if( jsonObjectGetNumber( ilsevent ) != 0.0 ) {
 762                 const char* desc = jsonObjectGetString( jsonObjectGetKeyConst( result, "desc" ));
 763                 if( !desc )
 764                         desc = "(No reason available)";    // failsafe; shouldn't happen
 765                 osrfLogInfo( OSRF_LOG_MARK, "Failure to reset timeout: %s", desc );
 766                 jsonObjectFree( result );
 767                 return 1;
 768         }
 769
 770         // Revise our local proxy for the timeout deadline
 771         // by a smallish fraction of the timeout interval
 772         const char* timeout = jsonObjectGetString( jsonObjectGetKeyConst( result, "payload" ));
 773         if( !timeout )
 774                 timeout = "1";   // failsafe; shouldn't happen
 775         time_next_reset = now + atoi( timeout ) / 15;
 776
 777         jsonObjectFree( result );
 778         return 0;     // Successfully reset timeout
 779 }
 780
 781 /**
 782         @brief Get the authkey string for the current application session, if any.
 783         @param ctx Pointer to the method context.
 784         @return Pointer to the cached authkey if found; otherwise NULL.
 785
 786         If present, the authkey string was cached from a previous method call.
 787 */
 788 static const char* getAuthkey( osrfMethodContext* ctx ) {
 789         if( ctx && ctx->session && ctx->session->userData ) {
 790                 const char* authkey = osrfHashGet( (osrfHash*) ctx->session->userData, "authkey" );
 791         // LFW recent changes mean the userData hash gets set up earlier, but
 792         // doesn't necessarily have an authkey yet
 793         if (!authkey)
 794             return NULL;
 795
 796                 // Possibly reset the authentication timeout to keep the login alive.  We do so
 797                 // no more than once per method call, and not at all if it has been only a short
 798                 // time since the last reset.
 799
 800                 // Here we reset explicitly, if at all.  We also implicitly reset the timeout
 801                 // whenever we call the "open-ils.auth.session.retrieve" method.
 802                 if( timeout_needs_resetting ) {
 803                         time_t now = time( NULL );
 804                         if( now >= time_next_reset && reset_timeout( authkey, now ) )
 805                                 authkey = NULL;    // timeout has apparently expired already
 806                 }
 807
 808                 timeout_needs_resetting = 0;
 809                 return authkey;
 810         }
 811         else
 812                 return NULL;
 813 }
 814
 815 /**
 816         @brief Implement the transaction.begin method.
 817         @param ctx Pointer to the method context.
 818         @return Zero if successful, or -1 upon error.
 819
 820         Start a transaction.  Save a transaction ID for future reference.
 821
 822         Method parameters:
 823         - authkey (PCRUD only)
 824
 825         Return to client: Transaction ID
 826 */
 827 int beginTransaction( osrfMethodContext* ctx ) {
 828         if(osrfMethodVerifyContext( ctx )) {
 829                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 830                 return -1;
 831         }
 832
 833         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
 834
 835         if( enforce_pcrud ) {
 836                 timeout_needs_resetting = 1;
 837                 const jsonObject* user = verifyUserPCRUD( ctx );
 838                 if( !user )
 839                         return -1;
 840         }
 841
 842         dbi_result result = dbi_conn_query( writehandle, "START TRANSACTION;" );
 843         if( !result ) {
 844                 const char* msg;
 845                 int errnum = dbi_conn_error( writehandle, &msg );
 846                 osrfLogError( OSRF_LOG_MARK, "%s: Error starting transaction: %d %s",
 847                         modulename, errnum, msg ? msg : "(No description available)" );
 848                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 849                         "osrfMethodException", ctx->request, "Error starting transaction" );
 850                 if( !oilsIsDBConnected( writehandle ))
 851                         osrfAppSessionPanic( ctx->session );
 852                 return -1;
 853         } else {
 854                 dbi_result_free( result );
 855                 setXactId( ctx );
 856                 jsonObject* ret = jsonNewObject( getXactId( ctx ) );
 857                 osrfAppRespondComplete( ctx, ret );
 858                 jsonObjectFree( ret );
 859
 860         }
 861
 862         if (tz) {
 863                 setenv("TZ",tz,1);
 864                 tzset();
 865                 dbi_result tz_res = dbi_conn_queryf( writehandle, "SET LOCAL timezone TO '%s'; -- cstore", tz );
 866                 if( !tz_res ) {
 867                         osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
 868                         if( !oilsIsDBConnected( writehandle )) {
 869                                 osrfAppSessionPanic( ctx->session );
 870                                 return -1;
 871                         }
 872                 } else {
 873                         dbi_result_free( tz_res );
 874                 }
 875         } else {
 876                 unsetenv("TZ");
 877                 tzset();
 878                 dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- no tz" );
 879                 if( !res ) {
 880                         osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
 881                         if( !oilsIsDBConnected( writehandle )) {
 882                                 osrfAppSessionPanic( ctx->session );
 883                                 return -1;
 884                         }
 885                 } else {
 886                         dbi_result_free( res );
 887                 }
 888         }
 889
 890         return 0;
 891 }
 892
 893 /**
 894         @brief Implement the savepoint.set method.
 895         @param ctx Pointer to the method context.
 896         @return Zero if successful, or -1 if not.
 897
 898         Issue a SAVEPOINT to the database server.
 899
 900         Method parameters:
 901         - authkey (PCRUD only)
 902         - savepoint name
 903
 904         Return to client: Savepoint name
 905 */
 906 int setSavepoint( osrfMethodContext* ctx ) {
 907         if(osrfMethodVerifyContext( ctx )) {
 908                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 909                 return -1;
 910         }
 911
 912         int spNamePos = 0;
 913         if( enforce_pcrud ) {
 914                 spNamePos = 1;
 915                 timeout_needs_resetting = 1;
 916                 const jsonObject* user = verifyUserPCRUD( ctx );
 917                 if( !user )
 918                         return -1;
 919         }
 920
 921         // Verify that a transaction is pending
 922         const char* trans_id = getXactId( ctx );
 923         if( NULL == trans_id ) {
 924                 osrfAppSessionStatus(
 925                         ctx->session,
 926                         OSRF_STATUS_INTERNALSERVERERROR,
 927                         "osrfMethodException",
 928                         ctx->request,
 929                         "No active transaction -- required for savepoints"
 930                 );
 931                 return -1;
 932         }
 933
 934         // Get the savepoint name from the method params
 935         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
 936
 937         if (!spName) {
 938                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.set called with no name");
 939                 return -1;
 940         }
 941
 942         char *safeSpName = _sanitize_savepoint_name( spName );
 943
 944         dbi_result result = dbi_conn_queryf( writehandle, "SAVEPOINT \"%s\";", safeSpName );
 945         free( safeSpName );
 946         if( !result ) {
 947                 const char* msg;
 948                 int errnum = dbi_conn_error( writehandle, &msg );
 949                 osrfLogError(
 950                         OSRF_LOG_MARK,
 951                         "%s: Error creating savepoint %s in transaction %s: %d %s",
 952                         modulename,
 953                         spName,
 954                         trans_id,
 955                         errnum,
 956                         msg ? msg : "(No description available)"
 957                 );
 958                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 959                         "osrfMethodException", ctx->request, "Error creating savepoint" );
 960                 if( !oilsIsDBConnected( writehandle ))
 961                         osrfAppSessionPanic( ctx->session );
 962                 return -1;
 963         } else {
 964                 dbi_result_free( result );
 965                 jsonObject* ret = jsonNewObject( spName );
 966                 osrfAppRespondComplete( ctx, ret );
 967                 jsonObjectFree( ret  );
 968                 return 0;
 969         }
 970 }
 971
 972 /**
 973         @brief Implement the savepoint.release method.
 974         @param ctx Pointer to the method context.
 975         @return Zero if successful, or -1 if not.
 976
 977         Issue a RELEASE SAVEPOINT to the database server.
 978
 979         Method parameters:
 980         - authkey (PCRUD only)
 981         - savepoint name
 982
 983         Return to client: Savepoint name
 984 */
 985 int releaseSavepoint( osrfMethodContext* ctx ) {
 986         if(osrfMethodVerifyContext( ctx )) {
 987                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 988                 return -1;
 989         }
 990
 991         int spNamePos = 0;
 992         if( enforce_pcrud ) {
 993                 spNamePos = 1;
 994                 timeout_needs_resetting = 1;
 995                 const jsonObject* user = verifyUserPCRUD( ctx );
 996                 if(  !user )
 997                         return -1;
 998         }
 999
1000         // Verify that a transaction is pending
1001         const char* trans_id = getXactId( ctx );
1002         if( NULL == trans_id ) {
1003                 osrfAppSessionStatus(
1004                         ctx->session,
1005                         OSRF_STATUS_INTERNALSERVERERROR,
1006                         "osrfMethodException",
1007                         ctx->request,
1008                         "No active transaction -- required for savepoints"
1009                 );
1010                 return -1;
1011         }
1012
1013         // Get the savepoint name from the method params
1014         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1015
1016         if (!spName) {
1017                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.release called with no name");
1018                 return -1;
1019         }
1020
1021         char *safeSpName = _sanitize_savepoint_name( spName );
1022
1023         dbi_result result = dbi_conn_queryf( writehandle, "RELEASE SAVEPOINT \"%s\";", safeSpName );
1024         free( safeSpName );
1025         if( !result ) {
1026                 const char* msg;
1027                 int errnum = dbi_conn_error( writehandle, &msg );
1028                 osrfLogError(
1029                         OSRF_LOG_MARK,
1030                         "%s: Error releasing savepoint %s in transaction %s: %d %s",
1031                         modulename,
1032                         spName,
1033                         trans_id,
1034                         errnum,
1035                         msg ? msg : "(No description available)"
1036                 );
1037                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1038                         "osrfMethodException", ctx->request, "Error releasing savepoint" );
1039                 if( !oilsIsDBConnected( writehandle ))
1040                         osrfAppSessionPanic( ctx->session );
1041                 return -1;
1042         } else {
1043                 dbi_result_free( result );
1044                 jsonObject* ret = jsonNewObject( spName );
1045                 osrfAppRespondComplete( ctx, ret );
1046                 jsonObjectFree( ret );
1047                 return 0;
1048         }
1049 }
1050
1051 /**
1052         @brief Implement the savepoint.rollback method.
1053         @param ctx Pointer to the method context.
1054         @return Zero if successful, or -1 if not.
1055
1056         Issue a ROLLBACK TO SAVEPOINT to the database server.
1057
1058         Method parameters:
1059         - authkey (PCRUD only)
1060         - savepoint name
1061
1062         Return to client: Savepoint name
1063 */
1064 int rollbackSavepoint( osrfMethodContext* ctx ) {
1065         if(osrfMethodVerifyContext( ctx )) {
1066                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1067                 return -1;
1068         }
1069
1070         int spNamePos = 0;
1071         if( enforce_pcrud ) {
1072                 spNamePos = 1;
1073                 timeout_needs_resetting = 1;
1074                 const jsonObject* user = verifyUserPCRUD( ctx );
1075                 if( !user )
1076                         return -1;
1077         }
1078
1079         // Verify that a transaction is pending
1080         const char* trans_id = getXactId( ctx );
1081         if( NULL == trans_id ) {
1082                 osrfAppSessionStatus(
1083                         ctx->session,
1084                         OSRF_STATUS_INTERNALSERVERERROR,
1085                         "osrfMethodException",
1086                         ctx->request,
1087                         "No active transaction -- required for savepoints"
1088                 );
1089                 return -1;
1090         }
1091
1092         // Get the savepoint name from the method params
1093         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1094
1095         if (!spName) {
1096                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.rollback called with no name");
1097                 return -1;
1098         }
1099
1100         char *safeSpName = _sanitize_savepoint_name( spName );
1101
1102         dbi_result result = dbi_conn_queryf( writehandle, "ROLLBACK TO SAVEPOINT \"%s\";", safeSpName );
1103         free( safeSpName );
1104         if( !result ) {
1105                 const char* msg;
1106                 int errnum = dbi_conn_error( writehandle, &msg );
1107                 osrfLogError(
1108                         OSRF_LOG_MARK,
1109                         "%s: Error rolling back savepoint %s in transaction %s: %d %s",
1110                         modulename,
1111                         spName,
1112                         trans_id,
1113                         errnum,
1114                         msg ? msg : "(No description available)"
1115                 );
1116                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1117                         "osrfMethodException", ctx->request, "Error rolling back savepoint" );
1118                 if( !oilsIsDBConnected( writehandle ))
1119                         osrfAppSessionPanic( ctx->session );
1120                 return -1;
1121         } else {
1122                 dbi_result_free( result );
1123                 jsonObject* ret = jsonNewObject( spName );
1124                 osrfAppRespondComplete( ctx, ret );
1125                 jsonObjectFree( ret );
1126                 return 0;
1127         }
1128 }
1129
1130 /**
1131         @brief Implement the transaction.commit method.
1132         @param ctx Pointer to the method context.
1133         @return Zero if successful, or -1 if not.
1134
1135         Issue a COMMIT to the database server.
1136
1137         Method parameters:
1138         - authkey (PCRUD only)
1139
1140         Return to client: Transaction ID.
1141 */
1142 int commitTransaction( osrfMethodContext* ctx ) {
1143         if(osrfMethodVerifyContext( ctx )) {
1144                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1145                 return -1;
1146         }
1147
1148         if( enforce_pcrud ) {
1149                 timeout_needs_resetting = 1;
1150                 const jsonObject* user = verifyUserPCRUD( ctx );
1151                 if( !user )
1152                         return -1;
1153         }
1154
1155         // Verify that a transaction is pending
1156         const char* trans_id = getXactId( ctx );
1157         if( NULL == trans_id ) {
1158                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1159                                 "osrfMethodException", ctx->request, "No active transaction to commit" );
1160                 return -1;
1161         }
1162
1163         dbi_result result = dbi_conn_query( writehandle, "COMMIT;" );
1164         if( !result ) {
1165                 const char* msg;
1166                 int errnum = dbi_conn_error( writehandle, &msg );
1167                 osrfLogError( OSRF_LOG_MARK, "%s: Error committing transaction: %d %s",
1168                         modulename, errnum, msg ? msg : "(No description available)" );
1169                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1170                         "osrfMethodException", ctx->request, "Error committing transaction" );
1171                 if( !oilsIsDBConnected( writehandle ))
1172                         osrfAppSessionPanic( ctx->session );
1173                 return -1;
1174         } else {
1175                 dbi_result_free( result );
1176                 jsonObject* ret = jsonNewObject( trans_id );
1177                 osrfAppRespondComplete( ctx, ret );
1178                 jsonObjectFree( ret );
1179                 clearXactId( ctx );
1180                 return 0;
1181         }
1182 }
1183
1184 /**
1185         @brief Implement the transaction.rollback method.
1186         @param ctx Pointer to the method context.
1187         @return Zero if successful, or -1 if not.
1188
1189         Issue a ROLLBACK to the database server.
1190
1191         Method parameters:
1192         - authkey (PCRUD only)
1193
1194         Return to client: Transaction ID
1195 */
1196 int rollbackTransaction( osrfMethodContext* ctx ) {
1197         if( osrfMethodVerifyContext( ctx )) {
1198                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1199                 return -1;
1200         }
1201
1202         if( enforce_pcrud ) {
1203                 timeout_needs_resetting = 1;
1204                 const jsonObject* user = verifyUserPCRUD( ctx );
1205                 if( !user )
1206                         return -1;
1207         }
1208
1209         // Verify that a transaction is pending
1210         const char* trans_id = getXactId( ctx );
1211         if( NULL == trans_id ) {
1212                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1213                                 "osrfMethodException", ctx->request, "No active transaction to roll back" );
1214                 return -1;
1215         }
1216
1217         dbi_result result = dbi_conn_query( writehandle, "ROLLBACK;" );
1218         if( !result ) {
1219                 const char* msg;
1220                 int errnum = dbi_conn_error( writehandle, &msg );
1221                 osrfLogError( OSRF_LOG_MARK, "%s: Error rolling back transaction: %d %s",
1222                         modulename, errnum, msg ? msg : "(No description available)" );
1223                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1224                         "osrfMethodException", ctx->request, "Error rolling back transaction" );
1225                 if( !oilsIsDBConnected( writehandle ))
1226                         osrfAppSessionPanic( ctx->session );
1227                 return -1;
1228         } else {
1229                 dbi_result_free( result );
1230                 jsonObject* ret = jsonNewObject( trans_id );
1231                 osrfAppRespondComplete( ctx, ret );
1232                 jsonObjectFree( ret );
1233                 clearXactId( ctx );
1234                 return 0;
1235         }
1236 }
1237
1238 /**
1239         @brief Implement the "search" method.
1240         @param ctx Pointer to the method context.
1241         @return Zero if successful, or -1 if not.
1242
1243         Method parameters:
1244         - authkey (PCRUD only)
1245         - WHERE clause, as jsonObject
1246         - Other SQL clause(s), as a JSON_HASH: joins, SELECT list, LIMIT, etc.
1247
1248         Return to client: rows of the specified class that satisfy a specified WHERE clause.
1249         Optionally flesh linked fields.
1250 */
1251 int doSearch( osrfMethodContext* ctx ) {
1252         if( osrfMethodVerifyContext( ctx )) {
1253                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1254                 return -1;
1255         }
1256
1257         if( enforce_pcrud )
1258                 timeout_needs_resetting = 1;
1259
1260         jsonObject* where_clause;
1261         jsonObject* rest_of_query;
1262
1263         if( enforce_pcrud ) {
1264                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1265                 rest_of_query = jsonObjectGetIndex( ctx->params, 2 );
1266         } else {
1267                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1268                 rest_of_query = jsonObjectGetIndex( ctx->params, 1 );
1269         }
1270
1271         if( !where_clause ) {
1272                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1273                 return -1;
1274         }
1275
1276         // Get the class metadata
1277         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1278         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1279
1280         // Do the query
1281         int err = 0;
1282         jsonObject* obj = doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1283         if( err ) {
1284                 osrfAppRespondComplete( ctx, NULL );
1285                 return -1;
1286         }
1287
1288         // doFieldmapperSearch() now takes care of our responding for us
1289 //      // Return each row to the client
1290 //      jsonObject* cur = 0;
1291 //      unsigned long res_idx = 0;
1292 //
1293 //      while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1294 //              // We used to discard based on perms here, but now that's
1295 //              // inside doFieldmapperSearch()
1296 //              osrfAppRespond( ctx, cur );
1297 //      }
1298
1299         jsonObjectFree( obj );
1300
1301         osrfAppRespondComplete( ctx, NULL );
1302         return 0;
1303 }
1304
1305 /**
1306         @brief Implement the "id_list" method.
1307         @param ctx Pointer to the method context.
1308         @param err Pointer through which to return an error code.
1309         @return Zero if successful, or -1 if not.
1310
1311         Method parameters:
1312         - authkey (PCRUD only)
1313         - WHERE clause, as jsonObject
1314         - Other SQL clause(s), as a JSON_HASH: joins, LIMIT, etc.
1315
1316         Return to client: The primary key values for all rows of the relevant class that
1317         satisfy a specified WHERE clause.
1318
1319         This method relies on the assumption that every class has a primary key consisting of
1320         a single column.
1321 */
1322 int doIdList( osrfMethodContext* ctx ) {
1323         if( osrfMethodVerifyContext( ctx )) {
1324                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1325                 return -1;
1326         }
1327
1328         if( enforce_pcrud )
1329                 timeout_needs_resetting = 1;
1330
1331         jsonObject* where_clause;
1332         jsonObject* rest_of_query;
1333
1334         // We use the where clause without change.  But we need to massage the rest of the
1335         // query, so we work with a copy of it instead of modifying the original.
1336
1337         if( enforce_pcrud ) {
1338                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1339                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 2 ) );
1340         } else {
1341                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1342                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 1 ) );
1343         }
1344
1345         if( !where_clause ) {
1346                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1347                 return -1;
1348         }
1349
1350         // Eliminate certain SQL clauses, if present.
1351         if( rest_of_query ) {
1352                 jsonObjectRemoveKey( rest_of_query, "select" );
1353                 jsonObjectRemoveKey( rest_of_query, "no_i18n" );
1354                 jsonObjectRemoveKey( rest_of_query, "flesh" );
1355                 jsonObjectRemoveKey( rest_of_query, "flesh_fields" );
1356         } else {
1357                 rest_of_query = jsonNewObjectType( JSON_HASH );
1358         }
1359
1360         jsonObjectSetKey( rest_of_query, "no_i18n", jsonNewBoolObject( 1 ) );
1361
1362         // Get the class metadata
1363         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1364         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1365
1366         // Build a SELECT list containing just the primary key,
1367         // i.e. like { "classname":["keyname"] }
1368         jsonObject* col_list_obj = jsonNewObjectType( JSON_ARRAY );
1369
1370         // Load array with name of primary key
1371         jsonObjectPush( col_list_obj, jsonNewObject( osrfHashGet( class_meta, "primarykey" ) ) );
1372         jsonObject* select_clause = jsonNewObjectType( JSON_HASH );
1373         jsonObjectSetKey( select_clause, osrfHashGet( class_meta, "classname" ), col_list_obj );
1374
1375         jsonObjectSetKey( rest_of_query, "select", select_clause );
1376
1377         // Do the query
1378         int err = 0;
1379         jsonObject* obj =
1380                 doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1381
1382         jsonObjectFree( rest_of_query );
1383         if( err ) {
1384                 osrfAppRespondComplete( ctx, NULL );
1385                 return -1;
1386         }
1387
1388         // Return each primary key value to the client
1389         jsonObject* cur;
1390         unsigned long res_idx = 0;
1391         while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1392                 // We used to discard based on perms here, but now that's
1393                 // inside doFieldmapperSearch()
1394                 osrfAppRespond( ctx,
1395                         oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
1396         }
1397
1398         jsonObjectFree( obj );
1399         osrfAppRespondComplete( ctx, NULL );
1400         return 0;
1401 }
1402
1403 /**
1404         @brief Verify that we have a valid class reference.
1405         @param ctx Pointer to the method context.
1406         @param param Pointer to the method parameters.
1407         @return 1 if the class reference is valid, or zero if it isn't.
1408
1409         The class of the method params must match the class to which the method id devoted.
1410         For PCRUD there are additional restrictions.
1411 */
1412 static int verifyObjectClass ( osrfMethodContext* ctx, const jsonObject* param ) {
1413
1414         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1415         osrfHash* class = osrfHashGet( method_meta, "class" );
1416
1417         // Compare the method's class to the parameters' class
1418         if( !param->classname || (strcmp( osrfHashGet(class, "classname"), param->classname ))) {
1419
1420                 // Oops -- they don't match.  Complain.
1421                 growing_buffer* msg = buffer_init( 128 );
1422                 buffer_fadd(
1423                         msg,
1424                         "%s: %s method for type %s was passed a %s",
1425                         modulename,
1426                         osrfHashGet( method_meta, "methodtype" ),
1427                         osrfHashGet( class, "classname" ),
1428                         param->classname ? param->classname : "(null)"
1429                 );
1430
1431                 char* m = buffer_release( msg );
1432                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_BADREQUEST, "osrfMethodException",
1433                                 ctx->request, m );
1434                 free( m );
1435
1436                 return 0;
1437         }
1438
1439         if( enforce_pcrud )
1440                 return verifyObjectPCRUD( ctx, class, param, 1 );
1441         else
1442                 return 1;
1443 }
1444
1445 /**
1446         @brief (PCRUD only) Verify that the user is properly logged in.
1447         @param ctx Pointer to the method context.
1448         @return If the user is logged in, a pointer to the user object from the authentication
1449         server; otherwise NULL.
1450 */
1451 static const jsonObject* verifyUserPCRUD( osrfMethodContext* ctx ) {
1452         return verifyUserPCRUDfull( ctx, 0 );
1453 }
1454
1455 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext* ctx, int anon_ok ) {
1456
1457         // Get the authkey (the first method parameter)
1458         const char* auth = jsonObjectGetString( jsonObjectGetIndex( ctx->params, 0 ) );
1459
1460         jsonObject* user = NULL;
1461
1462         // If we are /not/ in anonymous mode
1463         if( strcmp( "ANONYMOUS", auth ) ) {
1464                 // See if we have the same authkey, and a user object,
1465                 // locally cached from a previous call
1466                 const char* cached_authkey = getAuthkey( ctx );
1467                 if( cached_authkey && !strcmp( cached_authkey, auth ) ) {
1468                         const jsonObject* cached_user = getUserLogin( ctx );
1469                         if( cached_user )
1470                                 return cached_user;
1471                 }
1472
1473                 // We have no matching authentication data in the cache.  Authenticate from scratch.
1474                 jsonObject* auth_object = jsonNewObject( auth );
1475
1476                 // Fetch the user object from the authentication server
1477                 user = oilsUtilsQuickReq( "open-ils.auth", "open-ils.auth.session.retrieve", auth_object );
1478                 jsonObjectFree( auth_object );
1479
1480                 if( !user->classname || strcmp(user->classname, "au" )) {
1481
1482                         growing_buffer* msg = buffer_init( 128 );
1483                         buffer_fadd(
1484                                 msg,
1485                                 "%s: permacrud received a bad auth token: %s",
1486                                 modulename,
1487                                 auth
1488                         );
1489
1490                         char* m = buffer_release( msg );
1491                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_UNAUTHORIZED, "osrfMethodException",
1492                                         ctx->request, m );
1493
1494                         free( m );
1495                         jsonObjectFree( user );
1496                         user = NULL;
1497                 } else if( writeAuditInfo( ctx, oilsFMGetStringConst( user, "id" ), oilsFMGetStringConst( user, "wsid" ) ) ) {
1498                         // Failed to set audit information - But note that write_audit_info already set error information.
1499                         jsonObjectFree( user );
1500                         user = NULL;
1501                 }
1502
1503
1504         } else if ( anon_ok ) { // we /are/ (attempting to be) anonymous
1505                 user = jsonNewObjectType(JSON_ARRAY);
1506                 jsonObjectSetClass( user, "aou" );
1507                 oilsFMSetString(user, "id", "-1");
1508         }
1509
1510         setUserLogin( ctx, user );
1511         setAuthkey( ctx, auth );
1512
1513         // Allow ourselves up to a second before we have to reset the login timeout.
1514         // It would be nice to use some fraction of the timeout interval enforced by the
1515         // authentication server, but that value is not readily available at this point.
1516         // Instead, we use a conservative default interval.
1517         time_next_reset = time( NULL ) + 1;
1518
1519         return user;
1520 }
1521
1522 /**
1523         @brief For PCRUD: Determine whether the current user may access the current row.
1524         @param ctx Pointer to the method context.
1525         @param class Same as ctx->method->userData's item for key "class" except when called in recursive doFieldmapperSearch
1526         @param obj Pointer to the row being potentially accessed.
1527         @return 1 if access is permitted, or 0 if it isn't.
1528
1529         The @a obj parameter points to a JSON_HASH of column values, keyed on column name.
1530 */
1531 static int verifyObjectPCRUD ( osrfMethodContext* ctx, osrfHash *class, const jsonObject* obj, int rs_size ) {
1532
1533         dbhandle = writehandle;
1534
1535         // Figure out what class and method are involved
1536         osrfHash* method_metadata = (osrfHash*) ctx->method->userData;
1537         const char* method_type = osrfHashGet( method_metadata, "methodtype" );
1538
1539         if (!rs_size) {
1540                 int *rs_size_from_hash = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
1541                 if (rs_size_from_hash) {
1542                         rs_size = *rs_size_from_hash;
1543                         osrfLogDebug(OSRF_LOG_MARK, "used rs_size from request-scoped hash: %d", rs_size);
1544                 }
1545         }
1546
1547         // Set fetch to 1 in all cases except for inserts, meaning that for local or foreign
1548         // contexts we will do another lookup of the current row, even if we already have a
1549         // previously fetched row image, because the row image in hand may not include the
1550         // foreign key(s) that we need.
1551
1552         // This is a quick fix with a bludgeon.  There are ways to avoid the extra lookup,
1553         // but they aren't implemented yet.
1554
1555         int fetch = 0;
1556         if( *method_type == 's' || *method_type == 'i' ) {
1557                 method_type = "retrieve"; // search and id_list are equivalent to retrieve for this
1558                 fetch = 1;
1559         } else if( *method_type == 'u' || *method_type == 'd' ) {
1560                 fetch = 1; // MUST go to the db for the object for update and delete
1561         }
1562
1563         // In retrieve or search ONLY we allow anon. Later perm checks will fail as they should,
1564         // in the face of a fake user but required permissions.
1565         int anon_ok = 0;
1566         if( *method_type == 'r' )
1567                 anon_ok = 1;
1568
1569         // Get the appropriate permacrud entry from the IDL, depending on method type
1570         osrfHash* pcrud = osrfHashGet( osrfHashGet( class, "permacrud" ), method_type );
1571         if( !pcrud ) {
1572                 // No permacrud for this method type on this class
1573
1574                 growing_buffer* msg = buffer_init( 128 );
1575                 buffer_fadd(
1576                         msg,
1577                         "%s: %s on class %s has no permacrud IDL entry",
1578                         modulename,
1579                         osrfHashGet( method_metadata, "methodtype" ),
1580                         osrfHashGet( class, "classname" )
1581                 );
1582
1583                 char* m = buffer_release( msg );
1584                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_FORBIDDEN,
1585                                 "osrfMethodException", ctx->request, m );
1586
1587                 free( m );
1588
1589                 return 0;
1590         }
1591
1592         // Get the user id, and make sure the user is logged in
1593         const jsonObject* user = verifyUserPCRUDfull( ctx, anon_ok );
1594         if( !user )
1595                 return 0;    // Not logged in or anon?  No access.
1596
1597         int userid = atoi( oilsFMGetStringConst( user, "id" ) );
1598
1599         // Get a list of permissions from the permacrud entry.
1600         osrfStringArray* permission = osrfHashGet( pcrud, "permission" );
1601         if( permission->size == 0 ) {
1602                 osrfLogDebug(
1603                         OSRF_LOG_MARK,
1604                         "No permissions required for this action (class %s), passing through",
1605                         osrfHashGet(class, "classname")
1606                 );
1607                 return 1;
1608         }
1609
1610         // But, if there are perms and the user is anonymous ... FAIL
1611         if ( -1 == userid )
1612                 return 0;
1613
1614         // Build a list of org units that own the row.  This is fairly convoluted because there
1615         // are several different ways that an org unit may own the row, as defined by the
1616         // permacrud entry.
1617
1618         // Local context means that the row includes a foreign key pointing to actor.org_unit,
1619         // identifying an owning org_unit..
1620         osrfStringArray* local_context = osrfHashGet( pcrud, "local_context" );
1621
1622         // Foreign context adds a layer of indirection.  The row points to some other row that
1623         // an org unit may own.  The "jump" attribute, if present, adds another layer of
1624         // indirection.
1625         osrfHash* foreign_context = osrfHashGet( pcrud, "foreign_context" );
1626
1627         // The following string array stores the list of org units.  (We don't have a thingie
1628         // for storing lists of integers, so we fake it with a list of strings.)
1629         osrfStringArray* context_org_array = osrfNewStringArray( 1 );
1630
1631         const char* context_org = NULL;
1632     const char* pkey = NULL;
1633     jsonObject *param = NULL;
1634         const char* perm = NULL;
1635         int OK = 0;
1636         int i = 0;
1637         int err = 0;
1638         const char* pkey_value = NULL;
1639         if( str_is_true( osrfHashGet(pcrud, "global_required") ) ) {
1640                 // If the global_required attribute is present and true, then the only owning
1641                 // org unit is the root org unit, i.e. the one with no parent.
1642                 osrfLogDebug( OSRF_LOG_MARK,
1643                                 "global-level permissions required, fetching top of the org tree" );
1644
1645                 // no need to check perms for org tree root retrieval
1646                 osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1647                 // check for perm at top of org tree
1648                 const char* org_tree_root_id = org_tree_root( ctx );
1649                 osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1650
1651                 if( org_tree_root_id ) {
1652                         osrfStringArrayAdd( context_org_array, org_tree_root_id );
1653                         osrfLogDebug( OSRF_LOG_MARK, "top of the org tree is %s", org_tree_root_id );
1654                 } else  {
1655                         osrfStringArrayFree( context_org_array );
1656                         return 0;
1657                 }
1658
1659         } else {
1660                 // If the global_required attribute is absent or false, then we look for
1661                 // local and/or foreign context.  In order to find the relevant foreign
1662                 // keys, we must either read the relevant row from the database, or look at
1663                 // the image of the row that we already have in memory.
1664
1665                 // Even if we have an image of the row in memory, that image may not include the
1666                 // foreign key column(s) that we need.  So whenever possible, we do a fresh read
1667                 // of the row to make sure that we have what we need.
1668
1669             osrfLogDebug( OSRF_LOG_MARK, "global-level permissions not required, "
1670                                 "fetching context org ids" );
1671
1672         pkey = osrfHashGet( class, "primarykey" );
1673
1674                 if( !pkey ) {
1675                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1676                         // image that we already have.  If it doesn't have everything we need, too bad.
1677                         fetch = 0;
1678                         param = jsonObjectClone( obj );
1679                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1680                 } else if( obj->classname ) {
1681                         pkey_value = oilsFMGetStringConst( obj, pkey );
1682                         if( !fetch )
1683                                 param = jsonObjectClone( obj );
1684                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1685                                 pkey_value );
1686                 } else {
1687                         pkey_value = jsonObjectGetString( obj );
1688                         fetch = 1;
1689                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1690                                 "of %s and retrieving from the database", pkey_value );
1691                 }
1692
1693                 if( fetch ) {
1694                         // Fetch the row so that we can look at the foreign key(s)
1695                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1696                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1697                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1698                         jsonObjectFree( _tmp_params );
1699                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1700
1701                         param = jsonObjectExtractIndex( _list, 0 );
1702                         jsonObjectFree( _list );
1703
1704             fetch = 0;
1705                 }
1706
1707                 if( !param ) {
1708                         // The row doesn't exist.  Complain, and deny access.
1709                         osrfLogDebug( OSRF_LOG_MARK,
1710                                         "Object not found in the database with primary key %s of %s",
1711                                         pkey, pkey_value );
1712
1713                         growing_buffer* msg = buffer_init( 128 );
1714                         buffer_fadd(
1715                                 msg,
1716                                 "%s: no object found with primary key %s of %s",
1717                                 modulename,
1718                                 pkey,
1719                                 pkey_value
1720                         );
1721
1722                         char* m = buffer_release( msg );
1723                         osrfAppSessionStatus(
1724                                 ctx->session,
1725                                 OSRF_STATUS_INTERNALSERVERERROR,
1726                                 "osrfMethodException",
1727                                 ctx->request,
1728                                 m
1729                         );
1730
1731                         free( m );
1732                         return 0;
1733                 }
1734
1735                 if( local_context && local_context->size > 0 ) {
1736                         // The IDL provides a list of column names for the foreign keys denoting
1737                         // local context, i.e. columns identifying owing org units directly.  Look up
1738                         // the value of each one, and if it isn't null, add it to the list of org units.
1739                         osrfLogDebug( OSRF_LOG_MARK, "%d class-local context field(s) specified",
1740                                 local_context->size );
1741                         int i = 0;
1742                         const char* lcontext = NULL;
1743                         while ( (lcontext = osrfStringArrayGetString(local_context, i++)) ) {
1744                                 const char* fkey_value = oilsFMGetStringConst( param, lcontext );
1745                                 if( fkey_value ) {    // if not null
1746                                         osrfStringArrayAdd( context_org_array, fkey_value );
1747                                         osrfLogDebug(
1748                                                 OSRF_LOG_MARK,
1749                                                 "adding class-local field %s (value: %s) to the context org list",
1750                                                 lcontext,
1751                                                 osrfStringArrayGetString( context_org_array, context_org_array->size - 1 )
1752                                         );
1753                                 }
1754                         }
1755                 }
1756
1757                 if( foreign_context ) {
1758                         unsigned long class_count = osrfHashGetCount( foreign_context );
1759                         osrfLogDebug( OSRF_LOG_MARK, "%d foreign context classes(s) specified", class_count );
1760
1761                         if( class_count > 0 ) {
1762
1763                                 // The IDL provides a list of foreign key columns pointing to rows that
1764                                 // an org unit may own.  Follow each link, identify the owning org unit,
1765                                 // and add it to the list.
1766                                 osrfHash* fcontext = NULL;
1767                                 osrfHashIterator* class_itr = osrfNewHashIterator( foreign_context );
1768                                 while( (fcontext = osrfHashIteratorNext( class_itr )) ) {
1769                                         // For each class to which a foreign key points:
1770                                         const char* class_name = osrfHashIteratorKey( class_itr );
1771                                         osrfHash* fcontext = osrfHashGet( foreign_context, class_name );
1772
1773                                         osrfLogDebug(
1774                                                 OSRF_LOG_MARK,
1775                                                 "%d foreign context fields(s) specified for class %s",
1776                                                 ((osrfStringArray*)osrfHashGet(fcontext,"context"))->size,
1777                                                 class_name
1778                                         );
1779
1780                                         // Get the name of the key field in the foreign table
1781                                         const char* foreign_pkey = osrfHashGet( fcontext, "field" );
1782
1783                                         // Get the value of the foreign key pointing to the foreign table
1784                                         char* foreign_pkey_value =
1785                                                         oilsFMGetString( param, osrfHashGet( fcontext, "fkey" ));
1786                                         if( !foreign_pkey_value )
1787                                                 continue;    // Foreign key value is null; skip it
1788
1789                                         // Look up the row to which the foreign key points
1790                                         jsonObject* _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1791
1792                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1793                                         jsonObject* _list = doFieldmapperSearch(
1794                                                 ctx, osrfHashGet( oilsIDL(), class_name ), _tmp_params, NULL, &err );
1795                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1796
1797                                         jsonObject* _fparam = NULL;
1798                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1799                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1800
1801                                         jsonObjectFree( _tmp_params );
1802                                         jsonObjectFree( _list );
1803
1804                                         // At this point _fparam either points to the row identified by the
1805                                         // foreign key, or it's NULL (no such row found).
1806
1807                                         osrfStringArray* jump_list = osrfHashGet( fcontext, "jump" );
1808
1809                                         const char* bad_class = NULL;  // For noting failed lookups
1810                                         if( ! _fparam )
1811                                                 bad_class = class_name;    // Referenced row not found
1812                                         else if( jump_list ) {
1813                                                 // Follow a chain of rows, linked by foreign keys, to find an owner
1814                                                 const char* flink = NULL;
1815                                                 int k = 0;
1816                                                 while ( (flink = osrfStringArrayGetString(jump_list, k++)) && _fparam ) {
1817                                                         // For each entry in the jump list.  Each entry (i.e. flink) is
1818                                                         // the name of a foreign key column in the current row.
1819
1820                                                         // From the IDL, get the linkage information for the next jump
1821                                                         osrfHash* foreign_link_hash =
1822                                                                         oilsIDLFindPath( "/%s/links/%s", _fparam->classname, flink );
1823
1824                                                         // Get the class metadata for the class
1825                                                         // to which the foreign key points
1826                                                         osrfHash* foreign_class_meta = osrfHashGet( oilsIDL(),
1827                                                                         osrfHashGet( foreign_link_hash, "class" ));
1828
1829                                                         // Get the name of the referenced key of that class
1830                                                         foreign_pkey = osrfHashGet( foreign_link_hash, "key" );
1831
1832                                                         // Get the value of the foreign key pointing to that class
1833                                                         free( foreign_pkey_value );
1834                                                         foreign_pkey_value = oilsFMGetString( _fparam, flink );
1835                                                         if( !foreign_pkey_value )
1836                                                                 break;    // Foreign key is null; quit looking
1837
1838                                                         // Build a WHERE clause for the lookup
1839                                                         _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1840
1841                                                         // Do the lookup
1842                                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1843                                                         _list = doFieldmapperSearch( ctx, foreign_class_meta,
1844                                                                         _tmp_params, NULL, &err );
1845                                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1846
1847                                                         // Get the resulting row
1848                                                         jsonObjectFree( _fparam );
1849                                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1850                                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1851                                                         else {
1852                                                                 // Referenced row not found
1853                                                                 _fparam = NULL;
1854                                                                 bad_class = osrfHashGet( foreign_link_hash, "class" );
1855                                                         }
1856
1857                                                         jsonObjectFree( _tmp_params );
1858                                                         jsonObjectFree( _list );
1859                                                 }
1860                                         }
1861
1862                                         if( bad_class ) {
1863
1864                                                 // We had a foreign key pointing to such-and-such a row, but then
1865                                                 // we couldn't fetch that row.  The data in the database are in an
1866                                                 // inconsistent state; the database itself may even be corrupted.
1867                                                 growing_buffer* msg = buffer_init( 128 );
1868                                                 buffer_fadd(
1869                                                         msg,
1870                                                         "%s: no object of class %s found with primary key %s of %s",
1871                                                         modulename,
1872                                                         bad_class,
1873                                                         foreign_pkey,
1874                                                         foreign_pkey_value ? foreign_pkey_value : "(null)"
1875                                                 );
1876
1877                                                 char* m = buffer_release( msg );
1878                                                 osrfAppSessionStatus(
1879                                                         ctx->session,
1880                                                         OSRF_STATUS_INTERNALSERVERERROR,
1881                                                         "osrfMethodException",
1882                                                         ctx->request,
1883                                                         m
1884                                                 );
1885
1886                                                 free( m );
1887                                                 osrfHashIteratorFree( class_itr );
1888                                                 free( foreign_pkey_value );
1889                                                 jsonObjectFree( param );
1890
1891                                                 return 0;
1892                                         }
1893
1894                                         free( foreign_pkey_value );
1895
1896                                         if( _fparam ) {
1897                                                 // Examine each context column of the foreign row,
1898                                                 // and add its value to the list of org units.
1899                                                 int j = 0;
1900                                                 const char* foreign_field = NULL;
1901                                                 osrfStringArray* ctx_array = osrfHashGet( fcontext, "context" );
1902                                                 while ( (foreign_field = osrfStringArrayGetString( ctx_array, j++ )) ) {
1903                                                         osrfStringArrayAdd( context_org_array,
1904                                                                 oilsFMGetStringConst( _fparam, foreign_field ));
1905                                                         osrfLogDebug( OSRF_LOG_MARK,
1906                                                                 "adding foreign class %s field %s (value: %s) "
1907                                                                         "to the context org list",
1908                                                                 class_name,
1909                                                                 foreign_field,
1910                                                                 osrfStringArrayGetString(
1911                                                                         context_org_array, context_org_array->size - 1 )
1912                                                         );
1913                                                 }
1914
1915                                                 jsonObjectFree( _fparam );
1916                                         }
1917                                 }
1918
1919                                 osrfHashIteratorFree( class_itr );
1920                         }
1921                 }
1922         }
1923
1924     // If there is an owning_user attached to the action, we allow that user and users with
1925     // object perms on the object. CREATE can't use this. We only do this when we're not
1926     // ignoring object perms.
1927     char* owning_user_field = osrfHashGet( pcrud, "owning_user" );
1928     if (
1929         *method_type != 'c' &&
1930         (!str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) || // Always honor
1931         owning_user_field)
1932     ) {
1933         if (owning_user_field) { // see if we can short-cut by comparing the owner to the requestor
1934
1935             if (!param) { // We didn't get it during the context lookup
1936                             pkey = osrfHashGet( class, "primarykey" );
1937
1938                                 if( !pkey  ) {
1939                                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1940                                         // image that we already have.  If it doesn't have everything we need, too bad.
1941                                         fetch = 0;
1942                                         param = jsonObjectClone( obj );
1943                                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1944                                 } else if( obj->classname ) {
1945                                         pkey_value = oilsFMGetStringConst( obj, pkey );
1946                                         if( !fetch )
1947                                                 param = jsonObjectClone( obj );
1948                                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1949                                                 pkey_value );
1950                                 } else {
1951                                         pkey_value = jsonObjectGetString( obj );
1952                                         fetch = 1;
1953                                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1954                                                 "of %s and retrieving from the database", pkey_value );
1955                                 }
1956
1957                                 if( fetch ) {
1958                                         // Fetch the row so that we can look at the foreign key(s)
1959                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1960                                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1961                                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1962                                         jsonObjectFree( _tmp_params );
1963                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1964
1965                                         param = jsonObjectExtractIndex( _list, 0 );
1966                                         jsonObjectFree( _list );
1967                                 }
1968             }
1969
1970                         if( !param ) {
1971                                 // The row doesn't exist.  Complain, and deny access.
1972                                 osrfLogDebug( OSRF_LOG_MARK,
1973                                                 "Object not found in the database with primary key %s of %s",
1974                                                 pkey, pkey_value );
1975
1976                                 growing_buffer* msg = buffer_init( 128 );
1977                                 buffer_fadd(
1978                                         msg,
1979                                         "%s: no object found with primary key %s of %s",
1980                                         modulename,
1981                                         pkey,
1982                                         pkey_value
1983                                 );
1984
1985                                 char* m = buffer_release( msg );
1986                                 osrfAppSessionStatus(
1987                                         ctx->session,
1988                                         OSRF_STATUS_INTERNALSERVERERROR,
1989                                         "osrfMethodException",
1990                                         ctx->request,
1991                                         m
1992                                 );
1993
1994                                 free( m );
1995                                 return 0;
1996                         } else {
1997
1998                 int ownerid = atoi( oilsFMGetStringConst( param, owning_user_field ) );
1999
2000                 // Allow the owner to do whatever
2001                 if (ownerid == userid)
2002                     OK = 1;
2003                         }
2004                 }
2005
2006         i = 0;
2007         while(  !OK &&
2008                                 (perm = osrfStringArrayGetString(permission, i++)) &&
2009                                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms"))
2010                 ) {
2011                 dbi_result result;
2012
2013                         osrfLogDebug(
2014                                 OSRF_LOG_MARK,
2015                                 "Checking object permission [%s] for user %d "
2016                                                 "on object %s (class %s)",
2017                                 perm,
2018                                 userid,
2019                                 pkey_value,
2020                                 osrfHashGet( class, "classname" )
2021                         );
2022
2023                         result = dbi_conn_queryf(
2024                                 writehandle,
2025                                 "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s') AS has_perm;",
2026                                 userid,
2027                                 perm,
2028                                 osrfHashGet( class, "classname" ),
2029                                 pkey_value
2030                         );
2031
2032                         if( result ) {
2033                                 osrfLogDebug(
2034                                         OSRF_LOG_MARK,
2035                                         "Received a result for object permission [%s] "
2036                                                         "for user %d on object %s (class %s)",
2037                                         perm,
2038                                         userid,
2039                                         pkey_value,
2040                                         osrfHashGet( class, "classname" )
2041                                 );
2042
2043                                 if( dbi_result_first_row( result )) {
2044                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2045                                         const char* has_perm = jsonObjectGetString(
2046                                                         jsonObjectGetKeyConst( return_val, "has_perm" ));
2047
2048                                         osrfLogDebug(
2049                                                 OSRF_LOG_MARK,
2050                                                 "Status of object permission [%s] for user %d "
2051                                                                 "on object %s (class %s) is %s",
2052                                                 perm,
2053                                                 userid,
2054                                                 pkey_value,
2055                                                 osrfHashGet(class, "classname"),
2056                                                 has_perm
2057                                         );
2058
2059                                         if( *has_perm == 't' )
2060                                                 OK = 1;
2061                                         jsonObjectFree( return_val );
2062                                 }
2063
2064                                 dbi_result_free( result );
2065                                 if( OK )
2066                     break;
2067                         } else {
2068                                 const char* msg;
2069                                 int errnum = dbi_conn_error( writehandle, &msg );
2070                                 osrfLogWarning( OSRF_LOG_MARK,
2071                                         "Unable to call check object permissions: %d, %s",
2072                                         errnum, msg ? msg : "(No description available)" );
2073                                 if( !oilsIsDBConnected( writehandle ))
2074                                         osrfAppSessionPanic( ctx->session );
2075                         }
2076         }
2077     }
2078
2079         // For every combination of permission and context org unit: call a stored procedure
2080         // to determine if the user has this permission in the context of this org unit.
2081         // If the answer is yes at any point, then we're done, and the user has permission.
2082         // In other words permissions are additive.
2083         i = 0;
2084         while( !OK && (perm = osrfStringArrayGetString(permission, i++)) ) {
2085                 dbi_result result;
2086
2087         osrfStringArray* pcache = NULL;
2088         if (rs_size > perm_at_threshold) { // grab and cache locations of user perms
2089                         pcache = getPermLocationCache(ctx, perm);
2090
2091                         if (!pcache) {
2092                         pcache = osrfNewStringArray(0);
2093
2094                                 result = dbi_conn_queryf(
2095                                         writehandle,
2096                                         "SELECT permission.usr_has_perm_at_all(%d, '%s') AS at;",
2097                                         userid,
2098                                         perm
2099                                 );
2100
2101                                 if( result ) {
2102                                         osrfLogDebug(
2103                                                 OSRF_LOG_MARK,
2104                                                 "Received a result for permission [%s] for user %d",
2105                                                 perm,
2106                                                 userid
2107                                         );
2108
2109                                         if( dbi_result_first_row( result )) {
2110                             do {
2111                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2112                                                 osrfStringArrayAdd( pcache, jsonObjectGetString( jsonObjectGetKeyConst( return_val, "at" ) ) );
2113                                 jsonObjectFree( return_val );
2114                                             } while( dbi_result_next_row( result ));
2115
2116                                                 setPermLocationCache(ctx, perm, pcache);
2117                                         }
2118
2119                                         dbi_result_free( result );
2120                     }
2121                         }
2122         }
2123
2124                 int j = 0;
2125                 while( (context_org = osrfStringArrayGetString( context_org_array, j++ )) ) {
2126
2127             if (rs_size > perm_at_threshold) {
2128                 if (osrfStringArrayContains( pcache, context_org )) {
2129                     OK = 1;
2130                     break;
2131                 }
2132             }
2133
2134                         if(
2135                 pkey_value &&
2136                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) && // Always honor
2137                 (
2138                     !str_is_true( osrfHashGet(pcrud, "global_required") ) ||
2139                     osrfHashGet(pcrud, "owning_user")
2140                 )
2141             ) {
2142                                 osrfLogDebug(
2143                                         OSRF_LOG_MARK,
2144                                         "Checking object permission [%s] for user %d "
2145                                                         "on object %s (class %s) at org %d",
2146                                         perm,
2147                                         userid,
2148                                         pkey_value,
2149                                         osrfHashGet( class, "classname" ),
2150                                         atoi( context_org )
2151                                 );
2152
2153                                 result = dbi_conn_queryf(
2154                                         writehandle,
2155                                         "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s', %d) AS has_perm;",
2156                                         userid,
2157                                         perm,
2158                                         osrfHashGet( class, "classname" ),
2159                                         pkey_value,
2160                                         atoi( context_org )
2161                                 );
2162
2163                                 if( result ) {
2164                                         osrfLogDebug(
2165                                                 OSRF_LOG_MARK,
2166                                                 "Received a result for object permission [%s] "
2167                                                                 "for user %d on object %s (class %s) at org %d",
2168                                                 perm,
2169                                                 userid,
2170                                                 pkey_value,
2171                                                 osrfHashGet( class, "classname" ),
2172                                                 atoi( context_org )
2173                                         );
2174
2175                                         if( dbi_result_first_row( result )) {
2176                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2177                                                 const char* has_perm = jsonObjectGetString(
2178                                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2179
2180                                                 osrfLogDebug(
2181                                                         OSRF_LOG_MARK,
2182                                                         "Status of object permission [%s] for user %d "
2183                                                                         "on object %s (class %s) at org %d is %s",
2184                                                         perm,
2185                                                         userid,
2186                                                         pkey_value,
2187                                                         osrfHashGet(class, "classname"),
2188                                                         atoi(context_org),
2189                                                         has_perm
2190                                                 );
2191
2192                                                 if( *has_perm == 't' )
2193                                                         OK = 1;
2194                                                 jsonObjectFree( return_val );
2195                                         }
2196
2197                                         dbi_result_free( result );
2198                                         if( OK )
2199                                                 break;
2200                                 } else {
2201                                         const char* msg;
2202                                         int errnum = dbi_conn_error( writehandle, &msg );
2203                                         osrfLogWarning( OSRF_LOG_MARK,
2204                                                 "Unable to call check object permissions: %d, %s",
2205                                                 errnum, msg ? msg : "(No description available)" );
2206                                         if( !oilsIsDBConnected( writehandle ))
2207                                                 osrfAppSessionPanic( ctx->session );
2208                                 }
2209                         }
2210
2211             if (rs_size > perm_at_threshold) break;
2212
2213                         osrfLogDebug( OSRF_LOG_MARK,
2214                                         "Checking non-object permission [%s] for user %d at org %d",
2215                                         perm, userid, atoi(context_org) );
2216                         result = dbi_conn_queryf(
2217                                 writehandle,
2218                                 "SELECT permission.usr_has_perm(%d, '%s', %d) AS has_perm;",
2219                                 userid,
2220                                 perm,
2221                                 atoi( context_org )
2222                         );
2223
2224                         if( result ) {
2225                                 osrfLogDebug( OSRF_LOG_MARK,
2226                                         "Received a result for permission [%s] for user %d at org %d",
2227                                         perm, userid, atoi( context_org ));
2228                                 if( dbi_result_first_row( result )) {
2229                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2230                                         const char* has_perm = jsonObjectGetString(
2231                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2232                                         osrfLogDebug( OSRF_LOG_MARK,
2233                                                 "Status of permission [%s] for user %d at org %d is [%s]",
2234                                                 perm, userid, atoi( context_org ), has_perm );
2235                                         if( *has_perm == 't' )
2236                                                 OK = 1;
2237                                         jsonObjectFree( return_val );
2238                                 }
2239
2240                                 dbi_result_free( result );
2241                                 if( OK )
2242                                         break;
2243                         } else {
2244                                 const char* msg;
2245                                 int errnum = dbi_conn_error( writehandle, &msg );
2246                                 osrfLogWarning( OSRF_LOG_MARK, "Unable to call user object permissions: %d, %s",
2247                                         errnum, msg ? msg : "(No description available)" );
2248                                 if( !oilsIsDBConnected( writehandle ))
2249                                         osrfAppSessionPanic( ctx->session );
2250                         }
2251
2252                 }
2253
2254                 if( OK )
2255                         break;
2256         }
2257
2258         osrfStringArrayFree( context_org_array );
2259
2260         return OK;
2261 }
2262
2263 /**
2264         @brief Look up the root of the org_unit tree.
2265         @param ctx Pointer to the method context.
2266         @return The id of the root org unit, as a character string.
2267
2268         Query actor.org_unit where parent_ou is null, and return the id as a string.
2269
2270         This function assumes that there is only one root org unit, i.e. that we
2271         have a single tree, not a forest.
2272
2273         The calling code is responsible for freeing the returned string.
2274 */
2275 static const char* org_tree_root( osrfMethodContext* ctx ) {
2276
2277         static char cached_root_id[ 32 ] = "";  // extravagantly large buffer
2278         static time_t last_lookup_time = 0;
2279         time_t current_time = time( NULL );
2280
2281         if( cached_root_id[ 0 ] && ( current_time - last_lookup_time < 3600 ) ) {
2282                 // We successfully looked this up less than an hour ago.
2283                 // It's not likely to have changed since then.
2284                 return strdup( cached_root_id );
2285         }
2286         last_lookup_time = current_time;
2287
2288         int err = 0;
2289         jsonObject* where_clause = single_hash( "parent_ou", NULL );
2290         jsonObject* result = doFieldmapperSearch(
2291                 ctx, osrfHashGet( oilsIDL(), "aou" ), where_clause, NULL, &err );
2292         jsonObjectFree( where_clause );
2293
2294         jsonObject* tree_top = jsonObjectGetIndex( result, 0 );
2295
2296         if( !tree_top ) {
2297                 jsonObjectFree( result );
2298
2299                 growing_buffer* msg = buffer_init( 128 );
2300                 OSRF_BUFFER_ADD( msg, modulename );
2301                 OSRF_BUFFER_ADD( msg,
2302                                 ": Internal error, could not find the top of the org tree (parent_ou = NULL)" );
2303
2304                 char* m = buffer_release( msg );
2305                 osrfAppSessionStatus( ctx->session,
2306                                 OSRF_STATUS_INTERNALSERVERERROR, "osrfMethodException", ctx->request, m );
2307                 free( m );
2308
2309                 cached_root_id[ 0 ] = '\0';
2310                 return NULL;
2311         }
2312
2313         const char* root_org_unit_id = oilsFMGetStringConst( tree_top, "id" );
2314         osrfLogDebug( OSRF_LOG_MARK, "Top of the org tree is %s", root_org_unit_id );
2315
2316         strcpy( cached_root_id, root_org_unit_id );
2317         jsonObjectFree( result );
2318         return cached_root_id;
2319 }
2320
2321 /**
2322         @brief Create a JSON_HASH with a single key/value pair.
2323         @param key The key of the key/value pair.
2324         @param value the value of the key/value pair.
2325         @return Pointer to a newly created jsonObject of type JSON_HASH.
2326
2327         The value of the key/value is either a string or (if @a value is NULL) a null.
2328 */
2329 static jsonObject* single_hash( const char* key, const char* value ) {
2330         // Sanity check
2331         if( ! key ) key = "";
2332
2333         jsonObject* hash = jsonNewObjectType( JSON_HASH );
2334         jsonObjectSetKey( hash, key, jsonNewObject( value ) );
2335         return hash;
2336 }
2337
2338
2339 int doCreate( osrfMethodContext* ctx ) {
2340         if(osrfMethodVerifyContext( ctx )) {
2341                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2342                 return -1;
2343         }
2344
2345         if( enforce_pcrud )
2346                 timeout_needs_resetting = 1;
2347
2348         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2349         jsonObject* target = NULL;
2350         jsonObject* options = NULL;
2351
2352         if( enforce_pcrud ) {
2353                 target = jsonObjectGetIndex( ctx->params, 1 );
2354                 options = jsonObjectGetIndex( ctx->params, 2 );
2355         } else {
2356                 target = jsonObjectGetIndex( ctx->params, 0 );
2357                 options = jsonObjectGetIndex( ctx->params, 1 );
2358         }
2359
2360         if( !verifyObjectClass( ctx, target )) {
2361                 osrfAppRespondComplete( ctx, NULL );
2362                 return -1;
2363         }
2364
2365         osrfLogDebug( OSRF_LOG_MARK, "Object seems to be of the correct type" );
2366
2367         const char* trans_id = getXactId( ctx );
2368         if( !trans_id ) {
2369                 osrfLogError( OSRF_LOG_MARK, "No active transaction -- required for CREATE" );
2370
2371                 osrfAppSessionStatus(
2372                         ctx->session,
2373                         OSRF_STATUS_BADREQUEST,
2374                         "osrfMethodException",
2375                         ctx->request,
2376                         "No active transaction -- required for CREATE"
2377                 );
2378                 osrfAppRespondComplete( ctx, NULL );
2379                 return -1;
2380         }
2381
2382         // The following test is harmless but redundant.  If a class is
2383         // readonly, we don't register a create method for it.
2384         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
2385                 osrfAppSessionStatus(
2386                         ctx->session,
2387                         OSRF_STATUS_BADREQUEST,
2388                         "osrfMethodException",
2389                         ctx->request,
2390                         "Cannot INSERT readonly class"
2391                 );
2392                 osrfAppRespondComplete( ctx, NULL );
2393                 return -1;
2394         }
2395
2396         // Set the last_xact_id
2397         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
2398         if( index > -1 ) {
2399                 osrfLogDebug(OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
2400                         trans_id, target->classname, index);
2401                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
2402         }
2403
2404         osrfLogDebug( OSRF_LOG_MARK, "There is a transaction running..." );
2405
2406         dbhandle = writehandle;
2407
2408         osrfHash* fields = osrfHashGet( meta, "fields" );
2409         char* pkey       = osrfHashGet( meta, "primarykey" );
2410         char* seq        = osrfHashGet( meta, "sequence" );
2411
2412         growing_buffer* table_buf = buffer_init( 128 );
2413         growing_buffer* col_buf   = buffer_init( 128 );
2414         growing_buffer* val_buf   = buffer_init( 128 );
2415
2416         OSRF_BUFFER_ADD( table_buf, "INSERT INTO " );
2417         OSRF_BUFFER_ADD( table_buf, osrfHashGet( meta, "tablename" ));
2418         OSRF_BUFFER_ADD_CHAR( col_buf, '(' );
2419         buffer_add( val_buf,"VALUES (" );
2420
2421
2422         int first = 1;
2423         osrfHash* field = NULL;
2424         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
2425         while( (field = osrfHashIteratorNext( field_itr ) ) ) {
2426
2427                 const char* field_name = osrfHashIteratorKey( field_itr );
2428
2429                 if( str_is_true( osrfHashGet( field, "virtual" ) ) )
2430                         continue;
2431
2432                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
2433
2434                 char* value;
2435                 if( field_object && field_object->classname ) {
2436                         value = oilsFMGetString(
2437                                 field_object,
2438                                 (char*)oilsIDLFindPath( "/%s/primarykey", field_object->classname )
2439                         );
2440                 } else if( field_object && JSON_BOOL == field_object->type ) {
2441                         if( jsonBoolIsTrue( field_object ) )
2442                                 value = strdup( "t" );
2443                         else
2444                                 value = strdup( "f" );
2445                 } else {
2446                         value = jsonObjectToSimpleString( field_object );
2447                 }
2448
2449                 if( first ) {
2450                         first = 0;
2451                 } else {
2452                         OSRF_BUFFER_ADD_CHAR( col_buf, ',' );
2453                         OSRF_BUFFER_ADD_CHAR( val_buf, ',' );
2454                 }
2455
2456                 buffer_add( col_buf, field_name );
2457
2458                 if( !field_object || field_object->type == JSON_NULL ) {
2459                         buffer_add( val_buf, "DEFAULT" );
2460
2461                 } else if( !strcmp( get_primitive( field ), "number" )) {
2462                         const char* numtype = get_datatype( field );
2463                         if( !strcmp( numtype, "INT8" )) {
2464                                 buffer_fadd( val_buf, "%lld", atoll( value ));
2465
2466                         } else if( !strcmp( numtype, "INT" )) {
2467                                 buffer_fadd( val_buf, "%d", atoi( value ));
2468
2469                         } else if( !strcmp( numtype, "NUMERIC" )) {
2470                                 buffer_fadd( val_buf, "%f", atof( value ));
2471                         }
2472                 } else {
2473                         if( dbi_conn_quote_string( writehandle, &value )) {
2474                                 OSRF_BUFFER_ADD( val_buf, value );
2475
2476                         } else {
2477                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
2478                                 osrfAppSessionStatus(
2479                                         ctx->session,
2480                                         OSRF_STATUS_INTERNALSERVERERROR,
2481                                         "osrfMethodException",
2482                                         ctx->request,
2483                                         "Error quoting string -- please see the error log for more details"
2484                                 );
2485                                 free( value );
2486                                 buffer_free( table_buf );
2487                                 buffer_free( col_buf );
2488                                 buffer_free( val_buf );
2489                                 osrfAppRespondComplete( ctx, NULL );
2490                                 return -1;
2491                         }
2492                 }
2493
2494                 free( value );
2495         }
2496
2497         osrfHashIteratorFree( field_itr );
2498
2499         OSRF_BUFFER_ADD_CHAR( col_buf, ')' );
2500         OSRF_BUFFER_ADD_CHAR( val_buf, ')' );
2501
2502         char* table_str = buffer_release( table_buf );
2503         char* col_str   = buffer_release( col_buf );
2504         char* val_str   = buffer_release( val_buf );
2505         growing_buffer* sql = buffer_init( 128 );
2506         buffer_fadd( sql, "%s %s %s;", table_str, col_str, val_str );
2507         free( table_str );
2508         free( col_str );
2509         free( val_str );
2510
2511         char* query = buffer_release( sql );
2512
2513         osrfLogDebug( OSRF_LOG_MARK, "%s: Insert SQL [%s]", modulename, query );
2514
2515         jsonObject* obj = NULL;
2516         int rc = 0;
2517
2518         dbi_result result = dbi_conn_query( writehandle, query );
2519         if( !result ) {
2520                 obj = jsonNewObject( NULL );
2521                 const char* msg;
2522                 int errnum = dbi_conn_error( writehandle, &msg );
2523                 osrfLogError(
2524                         OSRF_LOG_MARK,
2525                         "%s ERROR inserting %s object using query [%s]: %d %s",
2526                         modulename,
2527                         osrfHashGet(meta, "fieldmapper"),
2528                         query,
2529                         errnum,
2530                         msg ? msg : "(No description available)"
2531                 );
2532                 osrfAppSessionStatus(
2533                         ctx->session,
2534                         OSRF_STATUS_INTERNALSERVERERROR,
2535                         "osrfMethodException",
2536                         ctx->request,
2537                         "INSERT error -- please see the error log for more details"
2538                 );
2539                 if( !oilsIsDBConnected( writehandle ))
2540                         osrfAppSessionPanic( ctx->session );
2541                 rc = -1;
2542         } else {
2543                 dbi_result_free( result );
2544
2545                 char* id = oilsFMGetString( target, pkey );
2546                 if( !id ) {
2547                         unsigned long long new_id = dbi_conn_sequence_last( writehandle, seq );
2548                         growing_buffer* _id = buffer_init( 10 );
2549                         buffer_fadd( _id, "%lld", new_id );
2550                         id = buffer_release( _id );
2551                 }
2552
2553                 // Find quietness specification, if present
2554                 const char* quiet_str = NULL;
2555                 if( options ) {
2556                         const jsonObject* quiet_obj = jsonObjectGetKeyConst( options, "quiet" );
2557                         if( quiet_obj )
2558                                 quiet_str = jsonObjectGetString( quiet_obj );
2559                 }
2560
2561                 if( str_is_true( quiet_str )) {  // if quietness is specified
2562                         obj = jsonNewObject( id );
2563                 }
2564                 else {
2565
2566                         // Fetch the row that we just inserted, so that we can return it to the client
2567                         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2568                         jsonObjectSetKey( where_clause, pkey, jsonNewObject( id ));
2569
2570                         int err = 0;
2571                         jsonObject* list = doFieldmapperSearch( ctx, meta, where_clause, NULL, &err );
2572                         if( err )
2573                                 rc = -1;
2574                         else
2575                                 obj = jsonObjectClone( jsonObjectGetIndex( list, 0 ));
2576
2577                         jsonObjectFree( list );
2578                         jsonObjectFree( where_clause );
2579                 }
2580
2581                 free( id );
2582         }
2583
2584         free( query );
2585         osrfAppRespondComplete( ctx, obj );
2586         jsonObjectFree( obj );
2587         return rc;
2588 }
2589
2590 /**
2591         @brief Implement the retrieve method.
2592         @param ctx Pointer to the method context.
2593         @param err Pointer through which to return an error code.
2594         @return If successful, a pointer to the result to be returned to the client;
2595         otherwise NULL.
2596
2597         From the method's class, fetch a row with a specified value in the primary key.  This
2598         method relies on the database design convention that a primary key consists of a single
2599         column.
2600
2601         Method parameters:
2602         - authkey (PCRUD only)
2603         - value of the primary key for the desired row, for building the WHERE clause
2604         - a JSON_HASH containing any other SQL clauses: select, join, etc.
2605
2606         Return to client: One row from the query.
2607 */
2608 int doRetrieve( osrfMethodContext* ctx ) {
2609         if(osrfMethodVerifyContext( ctx )) {
2610                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2611                 return -1;
2612         }
2613
2614         if( enforce_pcrud )
2615                 timeout_needs_resetting = 1;
2616
2617         int id_pos = 0;
2618         int order_pos = 1;
2619
2620         if( enforce_pcrud ) {
2621                 id_pos = 1;
2622                 order_pos = 2;
2623         }
2624
2625         // Get the class metadata
2626         osrfHash* class_def = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2627
2628         // Get the value of the primary key, from a method parameter
2629         const jsonObject* id_obj = jsonObjectGetIndex( ctx->params, id_pos );
2630
2631         osrfLogDebug(
2632                 OSRF_LOG_MARK,
2633                 "%s retrieving %s object with primary key value of %s",
2634                 modulename,
2635                 osrfHashGet( class_def, "fieldmapper" ),
2636                 jsonObjectGetString( id_obj )
2637         );
2638
2639         // Build a WHERE clause based on the key value
2640         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2641         jsonObjectSetKey(
2642                 where_clause,
2643                 osrfHashGet( class_def, "primarykey" ),  // name of key column
2644                 jsonObjectClone( id_obj )                // value of key column
2645         );
2646
2647         jsonObject* rest_of_query = jsonObjectGetIndex( ctx->params, order_pos );
2648
2649         // Do the query
2650         int err = 0;
2651         jsonObject* list = doFieldmapperSearch( ctx, class_def, where_clause, rest_of_query, &err );
2652
2653         jsonObjectFree( where_clause );
2654         if( err ) {
2655                 osrfAppRespondComplete( ctx, NULL );
2656                 return -1;
2657         }
2658
2659         jsonObject* obj = jsonObjectExtractIndex( list, 0 );
2660         jsonObjectFree( list );
2661
2662         if( enforce_pcrud ) {
2663                 // no result, skip this entirely
2664                 if(NULL != obj && !verifyObjectPCRUD( ctx, class_def, obj, 1 )) {
2665                         jsonObjectFree( obj );
2666
2667                         growing_buffer* msg = buffer_init( 128 );
2668                         OSRF_BUFFER_ADD( msg, modulename );
2669                         OSRF_BUFFER_ADD( msg, ": Insufficient permissions to retrieve object" );
2670
2671                         char* m = buffer_release( msg );
2672                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_NOTALLOWED, "osrfMethodException",
2673                                         ctx->request, m );
2674                         free( m );
2675
2676                         osrfAppRespondComplete( ctx, NULL );
2677                         return -1;
2678                 }
2679         }
2680
2681         // doFieldmapperSearch() now does the responding for us
2682         //osrfAppRespondComplete( ctx, obj );
2683         osrfAppRespondComplete( ctx, NULL );
2684
2685         jsonObjectFree( obj );
2686         return 0;
2687 }
2688
2689 /**
2690         @brief Translate a numeric value to a string representation for the database.
2691         @param field Pointer to the IDL field definition.
2692         @param value Pointer to a jsonObject holding the value of a field.
2693         @return Pointer to a newly allocated string.
2694
2695         The input object is typically a JSON_NUMBER, but it may be a JSON_STRING as long as
2696         its contents are numeric.  A non-numeric string is likely to result in invalid SQL.
2697
2698         If the datatype of the receiving field is not numeric, wrap the value in quotes.
2699
2700         The calling code is responsible for freeing the resulting string by calling free().
2701 */
2702 static char* jsonNumberToDBString( osrfHash* field, const jsonObject* value ) {
2703         growing_buffer* val_buf = buffer_init( 32 );
2704
2705         // If the value is a number and the DB field is numeric, no quotes needed
2706         if( value->type == JSON_NUMBER && !strcmp( get_primitive( field ), "number") ) {
2707                 buffer_fadd( val_buf, jsonObjectGetString( value ) );
2708         } else {
2709                 // Presumably this was really intended to be a string, so quote it
2710                 char* str = jsonObjectToSimpleString( value );
2711                 if( dbi_conn_quote_string( dbhandle, &str )) {
2712                         OSRF_BUFFER_ADD( val_buf, str );
2713                         free( str );
2714                 } else {
2715                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]", modulename, str );
2716                         free( str );
2717                         buffer_free( val_buf );
2718                         return NULL;
2719                 }
2720         }
2721
2722         return buffer_release( val_buf );
2723 }
2724
2725 static char* searchINPredicate( const char* class_alias, osrfHash* field,
2726                 jsonObject* node, const char* op, osrfMethodContext* ctx ) {
2727
2728         char* field_transform = searchFieldTransform( class_alias, field, node );
2729         if( ! field_transform )
2730                 return NULL;
2731
2732
2733         char* in_list = searchINList(field, node, op, ctx);
2734         if( !in_list )
2735                 return NULL;
2736
2737         growing_buffer* sql_buf = buffer_init( 32 );
2738         buffer_add( sql_buf, field_transform );
2739
2740         if( !op ) {
2741                 buffer_add( sql_buf, " IN (" );
2742         } else if( !strcasecmp( op,"not in" )) {
2743                 buffer_add( sql_buf, " NOT IN (" );
2744         } else {
2745                 buffer_add( sql_buf, " IN (" );
2746         }
2747
2748         buffer_add( sql_buf, in_list);
2749         OSRF_BUFFER_ADD_CHAR( sql_buf, ')' );
2750
2751         free(field_transform);
2752         free(in_list);
2753
2754         return buffer_release( sql_buf );
2755 }
2756
2757 static char* searchINList( osrfHash* field,
2758                 jsonObject* node, const char* op, osrfMethodContext* ctx ) {
2759         growing_buffer* sql_buf = buffer_init( 32 );
2760
2761         const jsonObject* local_node = node;
2762         if( local_node->type == JSON_HASH ) { // may be the case that the node tranforms the field
2763                 // if so, grab the "value" property
2764                 local_node = jsonObjectGetKeyConst( node, "value" );
2765                 if (!local_node) local_node = node;
2766         }
2767
2768         if( local_node->type == JSON_HASH ) {
2769                 // subquery predicate
2770
2771                 char* subpred = buildQuery( ctx, (jsonObject*) local_node, SUBSELECT );
2772                 if( ! subpred ) {
2773                         buffer_free( sql_buf );
2774                         return NULL;
2775                 }
2776
2777                 buffer_add( sql_buf, subpred );
2778                 free( subpred );
2779
2780         } else if( local_node->type == JSON_ARRAY ) {
2781                 // literal value list
2782                 int in_item_index = 0;
2783                 int in_item_first = 1;
2784                 const jsonObject* in_item;
2785                 while( (in_item = jsonObjectGetIndex( local_node, in_item_index++ )) ) {
2786
2787                         if( in_item_first )
2788                                 in_item_first = 0;
2789                         else
2790                                 buffer_add( sql_buf, ", " );
2791
2792                         // Sanity check
2793                         if( in_item->type != JSON_STRING && in_item->type != JSON_NUMBER ) {
2794                                 osrfLogError( OSRF_LOG_MARK,
2795                                                 "%s: Expected string or number within IN list; found %s",
2796                                                 modulename, json_type( in_item->type ) );
2797                                 buffer_free( sql_buf );
2798                                 return NULL;
2799                         }
2800
2801                         // Append the literal value -- quoted if not a number
2802                         if( JSON_NUMBER == in_item->type ) {
2803                                 char* val = jsonNumberToDBString( field, in_item );
2804                                 OSRF_BUFFER_ADD( sql_buf, val );
2805                                 free( val );
2806
2807                         } else if( !strcmp( get_primitive( field ), "number" )) {
2808                                 char* val = jsonNumberToDBString( field, in_item );
2809                                 OSRF_BUFFER_ADD( sql_buf, val );
2810                                 free( val );
2811
2812                         } else {
2813                                 char* key_string = jsonObjectToSimpleString( in_item );
2814                                 if( dbi_conn_quote_string( dbhandle, &key_string )) {
2815                                         OSRF_BUFFER_ADD( sql_buf, key_string );
2816                                         free( key_string );
2817                                 } else {
2818                                         osrfLogError( OSRF_LOG_MARK,
2819                                                         "%s: Error quoting key string [%s]", modulename, key_string );
2820                                         free( key_string );
2821                                         buffer_free( sql_buf );
2822                                         return NULL;
2823                                 }
2824                         }
2825                 }
2826
2827                 if( in_item_first ) {
2828                         osrfLogError(OSRF_LOG_MARK, "%s: Empty IN list", modulename );
2829                         buffer_free( sql_buf );
2830                         return NULL;
2831                 }
2832         } else {
2833                 osrfLogError( OSRF_LOG_MARK, "%s: Expected object or array for IN clause; found %s",
2834                         modulename, json_type( local_node->type ));
2835                 buffer_free( sql_buf );
2836                 return NULL;
2837         }
2838
2839         return buffer_release( sql_buf );
2840 }
2841
2842 // Receive a JSON_ARRAY representing a function call.  The first
2843 // entry in the array is the function name.  The rest are parameters.
2844 static char* searchValueTransform( const jsonObject* array ) {
2845
2846         if( array->size < 1 ) {
2847                 osrfLogError( OSRF_LOG_MARK, "%s: Empty array for value transform", modulename );
2848                 return NULL;
2849         }
2850
2851         // Get the function name
2852         jsonObject* func_item = jsonObjectGetIndex( array, 0 );
2853         if( func_item->type != JSON_STRING ) {
2854                 osrfLogError( OSRF_LOG_MARK, "%s: Error: expected function name, found %s",
2855                         modulename, json_type( func_item->type ));
2856                 return NULL;
2857         }
2858
2859         growing_buffer* sql_buf = buffer_init( 32 );
2860
2861         OSRF_BUFFER_ADD( sql_buf, jsonObjectGetString( func_item ) );
2862         OSRF_BUFFER_ADD( sql_buf, "( " );
2863
2864         // Get the parameters
2865         int func_item_index = 1;   // We already grabbed the zeroth entry
2866         while( (func_item = jsonObjectGetIndex( array, func_item_index++ )) ) {
2867
2868                 // Add a separator comma, if we need one
2869                 if( func_item_index > 2 )
2870                         buffer_add( sql_buf, ", " );
2871
2872                 // Add the current parameter
2873                 if( func_item->type == JSON_NULL ) {
2874                         buffer_add( sql_buf, "NULL" );
2875                 } else {
2876                         if( func_item->type == JSON_BOOL ) {
2877                                 if( jsonBoolIsTrue(func_item) ) {
2878                                         buffer_add( sql_buf, "TRUE" );
2879                                 } else {
2880                                         buffer_add( sql_buf, "FALSE" );
2881                                 }
2882                         } else {
2883                                 char* val = jsonObjectToSimpleString( func_item );
2884                                 if( dbi_conn_quote_string( dbhandle, &val )) {
2885                                         OSRF_BUFFER_ADD( sql_buf, val );
2886                                         free( val );
2887                                 } else {
2888                                         osrfLogError( OSRF_LOG_MARK,
2889                                                 "%s: Error quoting key string [%s]", modulename, val );
2890                                         buffer_free( sql_buf );
2891                                         free( val );
2892                                         return NULL;
2893                                 }
2894                         }
2895                 }
2896         }
2897
2898         buffer_add( sql_buf, " )" );
2899
2900         return buffer_release( sql_buf );
2901 }
2902
2903 static char* searchFunctionPredicate( const char* class_alias, osrfHash* field,
2904                 const jsonObject* node, const char* op ) {
2905
2906         if( ! is_good_operator( op ) ) {
2907                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
2908                 return NULL;
2909         }
2910
2911         char* val = searchValueTransform( node );
2912         if( !val )
2913                 return NULL;
2914
2915         const char* right_percent = "";
2916         const char* real_op       = op;
2917
2918         if( !strcasecmp( op, "startwith") ) {
2919                 real_op = "like";
2920                 right_percent = "|| '%'";
2921         }
2922
2923         growing_buffer* sql_buf = buffer_init( 32 );
2924         buffer_fadd(
2925                 sql_buf,
2926                 "\"%s\".%s %s %s%s",
2927                 class_alias,
2928                 osrfHashGet( field, "name" ),
2929                 real_op,
2930                 val,
2931                 right_percent
2932         );
2933
2934         free( val );
2935
2936         return buffer_release( sql_buf );
2937 }
2938
2939 // class_alias is a class name or other table alias
2940 // field is a field definition as stored in the IDL
2941 // node comes from the method parameter, and may represent an entry in the SELECT list
2942 static char* searchFieldTransform( const char* class_alias, osrfHash* field,
2943                 const jsonObject* node ) {
2944         growing_buffer* sql_buf = buffer_init( 32 );
2945
2946         if( node->type == JSON_HASH ) {
2947                 const char* field_transform = jsonObjectGetString(
2948                         jsonObjectGetKeyConst( node, "transform" ) );
2949                 const char* transform_subcolumn = jsonObjectGetString(
2950                         jsonObjectGetKeyConst( node, "result_field" ) );
2951
2952                 if( field_transform && transform_subcolumn ) {
2953                         if( ! is_identifier( transform_subcolumn ) ) {
2954                                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid subfield name: \"%s\"\n",
2955                                                 modulename, transform_subcolumn );
2956                                 buffer_free( sql_buf );
2957                                 return NULL;
2958                         }
2959                         OSRF_BUFFER_ADD_CHAR( sql_buf, '(' );   // enclose transform in parentheses
2960                 }
2961
2962                 if( field_transform ) {
2963
2964                         if( ! is_identifier( field_transform ) ) {
2965                                 osrfLogError( OSRF_LOG_MARK, "%s: Expected function name, found \"%s\"\n",
2966                                                 modulename, field_transform );
2967                                 buffer_free( sql_buf );
2968                                 return NULL;
2969                         }
2970
2971                         if( obj_is_true( jsonObjectGetKeyConst( node, "distinct" ) ) ) {
2972                                 buffer_fadd( sql_buf, "%s(DISTINCT \"%s\".%s",
2973                                         field_transform, class_alias, osrfHashGet( field, "name" ));
2974                         } else {
2975                                 buffer_fadd( sql_buf, "%s(\"%s\".%s",
2976                                         field_transform, class_alias, osrfHashGet( field, "name" ));
2977                         }
2978
2979                         const jsonObject* array = jsonObjectGetKeyConst( node, "params" );
2980
2981                         if( array ) {
2982                                 if( array->type != JSON_ARRAY ) {
2983                                         osrfLogError( OSRF_LOG_MARK,
2984                                                 "%s: Expected JSON_ARRAY for function params; found %s",
2985                                                 modulename, json_type( array->type ) );
2986                                         buffer_free( sql_buf );
2987                                         return NULL;
2988                                 }
2989                                 int func_item_index = 0;
2990                                 jsonObject* func_item;
2991                                 while( (func_item = jsonObjectGetIndex( array, func_item_index++ ))) {
2992
2993                                         char* val = jsonObjectToSimpleString( func_item );
2994
2995                                         if( !val ) {
2996                                                 buffer_add( sql_buf, ",NULL" );
2997                                         } else if( dbi_conn_quote_string( dbhandle, &val )) {
2998                                                 OSRF_BUFFER_ADD_CHAR( sql_buf, ',' );
2999                                                 OSRF_BUFFER_ADD( sql_buf, val );
3000                                         } else {
3001                                                 osrfLogError( OSRF_LOG_MARK,
3002                                                                 "%s: Error quoting key string [%s]", modulename, val );
3003                                                 free( val );
3004                                                 buffer_free( sql_buf );
3005                                                 return NULL;
3006                                         }
3007                                         free( val );
3008                                 }
3009                         }
3010
3011                         buffer_add( sql_buf, ")" );
3012
3013                         if( transform_subcolumn )
3014                                 buffer_fadd( sql_buf, ").\"%s\"", transform_subcolumn );
3015
3016                 } else {
3017                         buffer_fadd( sql_buf, "\"%s\".%s", class_alias, osrfHashGet( field, "name" ));
3018                 }
3019         } else {
3020                 buffer_fadd( sql_buf, "\"%s\".%s", class_alias, osrfHashGet( field, "name" ));
3021         }
3022
3023
3024         return buffer_release( sql_buf );
3025 }
3026
3027 static char* searchFieldTransformPredicate( const ClassInfo* class_info, osrfHash* field,
3028                 const jsonObject* node, const char* op ) {
3029
3030         if( ! is_good_operator( op ) ) {
3031                 osrfLogError( OSRF_LOG_MARK, "%s: Error: Invalid operator %s", modulename, op );
3032                 return NULL;
3033         }
3034
3035         char* field_transform = searchFieldTransform( class_info->alias, field, node );
3036         if( ! field_transform )
3037                 return NULL;
3038         char* value = NULL;
3039         int extra_parens = 0;   // boolean
3040
3041         const jsonObject* value_obj = jsonObjectGetKeyConst( node, "value" );
3042         if( ! value_obj ) {
3043                 value = searchWHERE( node, class_info, AND_OP_JOIN, NULL );
3044                 if( !value ) {
3045                         osrfLogError( OSRF_LOG_MARK, "%s: Error building condition for field transform",
3046                                 modulename );
3047                         free( field_transform );
3048                         return NULL;
3049                 }
3050                 extra_parens = 1;
3051         } else if( value_obj->type == JSON_ARRAY ) {
3052                 value = searchValueTransform( value_obj );
3053                 if( !value ) {
3054                         osrfLogError( OSRF_LOG_MARK,
3055                                 "%s: Error building value transform for field transform", modulename );
3056                         free( field_transform );
3057                         return NULL;
3058                 }
3059         } else if( value_obj->type == JSON_HASH ) {
3060                 value = searchWHERE( value_obj, class_info, AND_OP_JOIN, NULL );
3061                 if( !value ) {
3062                         osrfLogError( OSRF_LOG_MARK, "%s: Error building predicate for field transform",
3063                                 modulename );
3064                         free( field_transform );
3065                         return NULL;
3066                 }
3067                 extra_parens = 1;
3068         } else if( value_obj->type == JSON_NUMBER ) {
3069                 value = jsonNumberToDBString( field, value_obj );
3070         } else if( value_obj->type == JSON_NULL ) {
3071                 osrfLogError( OSRF_LOG_MARK,
3072                         "%s: Error building predicate for field transform: null value", modulename );
3073                 free( field_transform );
3074                 return NULL;
3075         } else if( value_obj->type == JSON_BOOL ) {
3076                 osrfLogError( OSRF_LOG_MARK,
3077                         "%s: Error building predicate for field transform: boolean value", modulename );
3078                 free( field_transform );
3079                 return NULL;
3080         } else {
3081                 if( !strcmp( get_primitive( field ), "number") ) {
3082                         value = jsonNumberToDBString( field, value_obj );
3083                 } else {
3084                         value = jsonObjectToSimpleString( value_obj );
3085                         if( !dbi_conn_quote_string( dbhandle, &value )) {
3086                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3087                                         modulename, value );
3088                                 free( value );
3089                                 free( field_transform );
3090                                 return NULL;
3091                         }
3092                 }
3093         }
3094
3095         const char* left_parens  = "";
3096         const char* right_parens = "";
3097
3098         if( extra_parens ) {
3099                 left_parens  = "(";
3100                 right_parens = ")";
3101         }
3102
3103         const char* right_percent = "";
3104         const char* real_op       = op;
3105
3106         if( !strcasecmp( op, "startwith") ) {
3107                 real_op = "like";
3108                 right_percent = "|| '%'";
3109         }
3110
3111         growing_buffer* sql_buf = buffer_init( 32 );
3112
3113         buffer_fadd(
3114                 sql_buf,
3115                 "%s%s %s %s %s%s %s%s",
3116                 left_parens,
3117                 field_transform,
3118                 real_op,
3119                 left_parens,
3120                 value,
3121                 right_percent,
3122                 right_parens,
3123                 right_parens
3124         );
3125
3126         free( value );
3127         free( field_transform );
3128
3129         return buffer_release( sql_buf );
3130 }
3131
3132 static char* searchSimplePredicate( const char* op, const char* class_alias,
3133                 osrfHash* field, const jsonObject* node ) {
3134
3135         if( ! is_good_operator( op ) ) {
3136                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
3137                 return NULL;
3138         }
3139
3140         char* val = NULL;
3141
3142         // Get the value to which we are comparing the specified column
3143         if( node->type != JSON_NULL ) {
3144                 if( node->type == JSON_NUMBER ) {
3145                         val = jsonNumberToDBString( field, node );
3146                 } else if( !strcmp( get_primitive( field ), "number" ) ) {
3147                         val = jsonNumberToDBString( field, node );
3148                 } else {
3149                         val = jsonObjectToSimpleString( node );
3150                 }
3151         }
3152
3153         if( val ) {
3154                 if( JSON_NUMBER != node->type && strcmp( get_primitive( field ), "number") ) {
3155                         // Value is not numeric; enclose it in quotes
3156                         if( !dbi_conn_quote_string( dbhandle, &val ) ) {
3157                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3158                                         modulename, val );
3159                                 free( val );
3160                                 return NULL;
3161                         }
3162                 }
3163         } else {
3164                 // Compare to a null value
3165                 val = strdup( "NULL" );
3166                 if( strcmp( op, "=" ))
3167                         op = "IS NOT";
3168                 else
3169                         op = "IS";
3170         }
3171
3172         const char* right_percent = "";
3173         const char* real_op       = op;
3174
3175         if( !strcasecmp( op, "startwith") ) {
3176                 real_op = "like";
3177                 right_percent = "|| '%'";
3178         }
3179
3180         growing_buffer* sql_buf = buffer_init( 32 );
3181         buffer_fadd( sql_buf, "\"%s\".%s %s %s%s", class_alias, osrfHashGet(field, "name"), real_op, val, right_percent );
3182         char* pred = buffer_release( sql_buf );
3183
3184         free( val );
3185
3186         return pred;
3187 }
3188
3189 static char* searchBETWEENRange( osrfHash* field, const jsonObject* node ) {
3190
3191         const jsonObject* local_node = node;
3192         if( node->type == JSON_HASH ) { // will be the case if the node tranforms the field
3193                 local_node = jsonObjectGetKeyConst( node, "value" );
3194                 if (!local_node) local_node = node;
3195         }
3196
3197         const jsonObject* x_node = jsonObjectGetIndex( local_node, 0 );
3198         const jsonObject* y_node = jsonObjectGetIndex( local_node, 1 );
3199
3200         if( NULL == y_node ) {
3201                 osrfLogError( OSRF_LOG_MARK, "%s: Not enough operands for BETWEEN operator", modulename );
3202                 return NULL;
3203         } else if( NULL != jsonObjectGetIndex( local_node, 2 ) ) {
3204                 osrfLogError( OSRF_LOG_MARK, "%s: Too many operands for BETWEEN operator", modulename );
3205                 return NULL;
3206         }
3207
3208         char* x_string;
3209         char* y_string;
3210
3211         if( !strcmp( get_primitive( field ), "number") ) {
3212                 x_string = jsonNumberToDBString( field, x_node );
3213                 y_string = jsonNumberToDBString( field, y_node );
3214
3215         } else {
3216                 x_string = jsonObjectToSimpleString( x_node );
3217                 y_string = jsonObjectToSimpleString( y_node );
3218                 if( !(dbi_conn_quote_string( dbhandle, &x_string )
3219                         && dbi_conn_quote_string( dbhandle, &y_string )) ) {
3220                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key strings [%s] and [%s]",
3221                                         modulename, x_string, y_string );
3222                         free( x_string );
3223                         free( y_string );
3224                         return NULL;
3225                 }
3226         }
3227
3228         growing_buffer* sql_buf = buffer_init( 32 );
3229         buffer_fadd( sql_buf, "%s AND %s", x_string, y_string );
3230         free( x_string );
3231         free( y_string );
3232
3233         return buffer_release( sql_buf );
3234 }
3235
3236 static char* searchBETWEENPredicate( const char* class_alias,
3237                 osrfHash* field, const jsonObject* node ) {
3238
3239         char* field_transform = searchFieldTransform( class_alias, field, node );
3240         if( ! field_transform )
3241                 return NULL;
3242
3243         char* between_range = searchBETWEENRange(field, node);
3244
3245         if( NULL == between_range )
3246                 return NULL;
3247
3248         growing_buffer* sql_buf = buffer_init( 32 );
3249         buffer_fadd( sql_buf, "%s BETWEEN %s", field_transform, between_range);
3250
3251         free(field_transform);
3252         free(between_range);
3253
3254         return buffer_release( sql_buf );
3255 }
3256
3257 static char* searchPredicate( const ClassInfo* class_info, osrfHash* field,
3258                                                           jsonObject* node, osrfMethodContext* ctx ) {
3259
3260         char* pred = NULL;
3261         if( node->type == JSON_ARRAY ) { // equality IN search
3262                 pred = searchINPredicate( class_info->alias, field, node, NULL, ctx );
3263         } else if( node->type == JSON_HASH ) { // other search
3264                 jsonIterator* pred_itr = jsonNewIterator( node );
3265                 if( !jsonIteratorHasNext( pred_itr ) ) {
3266                         osrfLogError( OSRF_LOG_MARK, "%s: Empty predicate for field \"%s\"",
3267                                         modulename, osrfHashGet(field, "name" ));
3268                 } else {
3269                         jsonObject* pred_node = jsonIteratorNext( pred_itr );
3270
3271                         // Verify that there are no additional predicates
3272                         if( jsonIteratorHasNext( pred_itr ) ) {
3273                                 osrfLogError( OSRF_LOG_MARK, "%s: Multiple predicates for field \"%s\"",
3274                                                 modulename, osrfHashGet(field, "name" ));
3275                         } else if( !(strcasecmp( pred_itr->key,"between" )) ) {
3276                                 pred = searchBETWEENPredicate( class_info->alias, field, pred_node );
3277                         } else if( !(strcasecmp( pred_itr->key,"in" ))
3278                                         || !(strcasecmp( pred_itr->key,"not in" )) ) {
3279                                 pred = searchINPredicate( class_info->alias, field, pred_node, pred_itr->key, ctx );
3280                         } else if( pred_node->type == JSON_ARRAY ) {
3281                                 pred = searchFunctionPredicate(
3282                                         class_info->alias, field, pred_node, pred_itr->key );
3283                         } else if( pred_node->type == JSON_HASH ) {
3284                                 pred = searchFieldTransformPredicate(
3285                                         class_info, field, pred_node, pred_itr->key );
3286                         } else {
3287                                 pred = searchSimplePredicate( pred_itr->key, class_info->alias, field, pred_node );
3288                         }
3289                 }
3290                 jsonIteratorFree( pred_itr );
3291
3292         } else if( node->type == JSON_NULL ) { // IS NULL search
3293                 growing_buffer* _p = buffer_init( 64 );
3294                 buffer_fadd(
3295                         _p,
3296                         "\"%s\".%s IS NULL",
3297                         class_info->alias,
3298                         osrfHashGet( field, "name" )
3299                 );
3300                 pred = buffer_release( _p );
3301         } else { // equality search
3302                 pred = searchSimplePredicate( "=", class_info->alias, field, node );
3303         }
3304
3305         return pred;
3306
3307 }
3308
3309
3310 /*
3311
3312 join : {
3313         acn : {
3314                 field : record,
3315                 fkey : id
3316                 type : left
3317                 filter_op : or
3318                 filter : { ... },
3319                 join : {
3320                         acp : {
3321                                 field : call_number,
3322                                 fkey : id,
3323                                 filter : { ... },
3324                         },
3325                 },
3326         },
3327         mrd : {
3328                 field : record,
3329                 type : inner
3330                 fkey : id,
3331                 filter : { ... },
3332         }
3333 }
3334
3335   Or, to specify join order:
3336
3337 join : [
3338     {mrd:{field:'record', type:'inner'}},
3339     {acn:{field:'record', type:'left'}}
3340 ]
3341
3342 */
3343
3344 static char* searchJOIN( const jsonObject* join_hash, const ClassInfo* left_info ) {
3345
3346         jsonObject* working_hash;
3347         jsonObject* freeable_hash = NULL;
3348
3349         jsonObject* working_array;
3350         jsonObject* freeable_array = NULL;
3351
3352         if( join_hash->type == JSON_ARRAY ) {
3353                 working_array = (jsonObject*)join_hash;
3354         } else {
3355                 working_array = jsonNewObjectType( JSON_ARRAY );
3356
3357                 if( join_hash->type == JSON_HASH ) {
3358                         working_hash = (jsonObject*)join_hash;
3359                 } else if( join_hash->type == JSON_STRING ) {
3360                     freeable_array = working_array;
3361                         // turn it into a JSON_HASH by creating a wrapper
3362                         // around a copy of the original
3363                         const char* _tmp = jsonObjectGetString( join_hash );
3364                         freeable_hash = jsonNewObjectType( JSON_HASH );
3365                         jsonObjectSetKey( freeable_hash, _tmp, NULL );
3366                         working_hash = freeable_hash;
3367                 } else {
3368                         osrfLogError(
3369                                 OSRF_LOG_MARK,
3370                                 "%s: JOIN failed; expected JSON object type not found",
3371                                 modulename
3372                         );
3373                         return NULL;
3374                 }
3375
3376                 jsonObjectPush( working_array, working_hash );
3377         }
3378
3379         growing_buffer* join_buf = buffer_init( 128 );
3380         const char* leftclass = left_info->class_name;
3381
3382         unsigned long order_idx = 0;
3383         while(( working_hash = jsonObjectGetIndex( working_array, order_idx++ ) )) {
3384
3385             jsonObject* freeable_subhash = NULL;
3386                 if( working_hash->type == JSON_STRING ) {
3387                         // turn it into a JSON_HASH by creating a wrapper
3388                         // around a copy of the original
3389                         const char* _inner_tmp = jsonObjectGetString( working_hash );
3390                         freeable_subhash = jsonNewObjectType( JSON_HASH );
3391                         jsonObjectSetKey( freeable_subhash, _inner_tmp, NULL );
3392                         working_hash = freeable_subhash;
3393                 }
3394
3395                 jsonObject* snode = NULL;
3396                 jsonIterator* search_itr = jsonNewIterator( working_hash );
3397
3398                 while ( (snode = jsonIteratorNext( search_itr )) ) {
3399                         const char* right_alias = search_itr->key;
3400                         const char* class =
3401                                         jsonObjectGetString( jsonObjectGetKeyConst( snode, "class" ) );
3402                         if( ! class )
3403                                 class = right_alias;
3404
3405                         const ClassInfo* right_info = add_joined_class( right_alias, class );
3406                         if( !right_info ) {
3407                                 osrfLogError(
3408                                         OSRF_LOG_MARK,
3409                                         "%s: JOIN failed.  Class \"%s\" not resolved in IDL",
3410                                         modulename,
3411                                         search_itr->key
3412                                 );
3413                                 jsonIteratorFree( search_itr );
3414                                 buffer_free( join_buf );
3415                                 if( freeable_subhash )
3416                                         jsonObjectFree( freeable_subhash );
3417                                 if( freeable_hash )
3418                                         jsonObjectFree( freeable_hash );
3419                                 if( freeable_array )
3420                                         jsonObjectFree( freeable_array );
3421                                 return NULL;
3422                         }
3423                         osrfHash* links = right_info->links;
3424                         const char* table  = right_info->source_def;
3425
3426                         const char* fkey  = jsonObjectGetString( jsonObjectGetKeyConst( snode, "fkey" ) );
3427                         const char* field = jsonObjectGetString( jsonObjectGetKeyConst( snode, "field" ) );
3428
3429                         if( field && !fkey ) {
3430                                 // Look up the corresponding join column in the IDL.
3431                                 // The link must be defined in the child table,
3432                                 // and point to the right parent table.
3433                                 osrfHash* idl_link = (osrfHash*) osrfHashGet( links, field );
3434                                 const char* reltype = NULL;
3435                                 const char* other_class = NULL;
3436                                 reltype = osrfHashGet( idl_link, "reltype" );
3437                                 if( reltype && strcmp( reltype, "has_many" ) )
3438                                         other_class = osrfHashGet( idl_link, "class" );
3439                                 if( other_class && !strcmp( other_class, leftclass ) )
3440                                         fkey = osrfHashGet( idl_link, "key" );
3441                                 if( !fkey ) {
3442                                         osrfLogError(
3443                                                 OSRF_LOG_MARK,
3444                                                 "%s: JOIN failed.  No link defined from %s.%s to %s",
3445                                                 modulename,
3446                                                 class,
3447                                                 field,
3448                                                 leftclass
3449                                         );
3450                                         buffer_free( join_buf );
3451                                         if( freeable_subhash )
3452                                                 jsonObjectFree( freeable_subhash );
3453                                         if( freeable_hash )
3454                                                 jsonObjectFree( freeable_hash );
3455                                         if( freeable_array )
3456                                                 jsonObjectFree( freeable_array );
3457                                         jsonIteratorFree( search_itr );
3458                                         return NULL;
3459                                 }
3460
3461                         } else if( !field && fkey ) {
3462                                 // Look up the corresponding join column in the IDL.
3463                                 // The link must be defined in the child table,
3464                                 // and point to the right parent table.
3465                                 osrfHash* left_links = left_info->links;
3466                                 osrfHash* idl_link = (osrfHash*) osrfHashGet( left_links, fkey );
3467                                 const char* reltype = NULL;
3468                                 const char* other_class = NULL;
3469                                 reltype = osrfHashGet( idl_link, "reltype" );
3470                                 if( reltype && strcmp( reltype, "has_many" ) )
3471                                         other_class = osrfHashGet( idl_link, "class" );
3472                                 if( other_class && !strcmp( other_class, class ) )
3473                                         field = osrfHashGet( idl_link, "key" );
3474                                 if( !field ) {
3475                                         osrfLogError(
3476                                                 OSRF_LOG_MARK,
3477                                                 "%s: JOIN failed.  No link defined from %s.%s to %s",
3478                                                 modulename,
3479                                                 leftclass,
3480                                                 fkey,
3481                                                 class
3482                                         );
3483                                         buffer_free( join_buf );
3484                                         if( freeable_subhash )
3485                                                 jsonObjectFree( freeable_subhash );
3486                                         if( freeable_hash )
3487                                                 jsonObjectFree( freeable_hash );
3488                                         if( freeable_array )
3489                                                 jsonObjectFree( freeable_array );
3490                                         jsonIteratorFree( search_itr );
3491                                         return NULL;
3492                                 }
3493
3494                         } else if( !field && !fkey ) {
3495                                 osrfHash* left_links = left_info->links;
3496
3497                                 // For each link defined for the left class:
3498                                 // see if the link references the joined class
3499                                 osrfHashIterator* itr = osrfNewHashIterator( left_links );
3500                                 osrfHash* curr_link = NULL;
3501                                 while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3502                                         const char* other_class = osrfHashGet( curr_link, "class" );
3503                                         if( other_class && !strcmp( other_class, class ) ) {
3504
3505                                                 // In the IDL, the parent class doesn't always know then names of the child
3506                                                 // columns that are pointing to it, so don't use that end of the link
3507                                                 const char* reltype = osrfHashGet( curr_link, "reltype" );
3508                                                 if( reltype && strcmp( reltype, "has_many" ) ) {
3509                                                         // Found a link between the classes
3510                                                         fkey = osrfHashIteratorKey( itr );
3511                                                         field = osrfHashGet( curr_link, "key" );
3512                                                         break;
3513                                                 }
3514                                         }
3515                                 }
3516                                 osrfHashIteratorFree( itr );
3517
3518                                 if( !field || !fkey ) {
3519                                         // Do another such search, with the classes reversed
3520
3521                                         // For each link defined for the joined class:
3522                                         // see if the link references the left class
3523                                         osrfHashIterator* itr = osrfNewHashIterator( links );
3524                                         osrfHash* curr_link = NULL;
3525                                         while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3526                                                 const char* other_class = osrfHashGet( curr_link, "class" );
3527                                                 if( other_class && !strcmp( other_class, leftclass ) ) {
3528
3529                                                         // In the IDL, the parent class doesn't know then names of the child
3530                                                         // columns that are pointing to it, so don't use that end of the link
3531                                                         const char* reltype = osrfHashGet( curr_link, "reltype" );
3532                                                         if( reltype && strcmp( reltype, "has_many" ) ) {
3533                                                                 // Found a link between the classes
3534                                                                 field = osrfHashIteratorKey( itr );
3535                                                                 fkey = osrfHashGet( curr_link, "key" );
3536                                                                 break;
3537                                                         }
3538                                                 }
3539                                         }
3540                                         osrfHashIteratorFree( itr );
3541                                 }
3542
3543                                 if( !field || !fkey ) {
3544                                         osrfLogError(
3545                                                 OSRF_LOG_MARK,
3546                                                 "%s: JOIN failed.  No link defined between %s and %s",
3547                                                 modulename,
3548                                                 leftclass,
3549                                                 class
3550                                         );
3551                                         buffer_free( join_buf );
3552                                         if( freeable_subhash )
3553                                                 jsonObjectFree( freeable_subhash );
3554                                         if( freeable_hash )
3555                                                 jsonObjectFree( freeable_hash );
3556                                         if( freeable_array )
3557                                                 jsonObjectFree( freeable_array );
3558                                         jsonIteratorFree( search_itr );
3559                                         return NULL;
3560                                 }
3561                         }
3562
3563                         const char* type = jsonObjectGetString( jsonObjectGetKeyConst( snode, "type" ) );
3564                         if( type ) {
3565                                 if( !strcasecmp( type,"left" )) {
3566                                         buffer_add( join_buf, " LEFT JOIN" );
3567                                 } else if( !strcasecmp( type,"right" )) {
3568                                         buffer_add( join_buf, " RIGHT JOIN" );
3569                                 } else if( !strcasecmp( type,"full" )) {
3570                                         buffer_add( join_buf, " FULL JOIN" );
3571                                 } else {
3572                                         buffer_add( join_buf, " INNER JOIN" );
3573                                 }
3574                         } else {
3575                                 buffer_add( join_buf, " INNER JOIN" );
3576                         }
3577
3578                         buffer_fadd( join_buf, " %s AS \"%s\" ON ( \"%s\".%s = \"%s\".%s",
3579                                                 table, right_alias, right_alias, field, left_info->alias, fkey );
3580
3581                         // Add any other join conditions as specified by "filter"
3582                         const jsonObject* filter = jsonObjectGetKeyConst( snode, "filter" );
3583                         if( filter ) {
3584                                 const char* filter_op = jsonObjectGetString(
3585                                         jsonObjectGetKeyConst( snode, "filter_op" ) );
3586                                 if( filter_op && !strcasecmp( "or",filter_op )) {
3587                                         buffer_add( join_buf, " OR " );
3588                                 } else {
3589                                         buffer_add( join_buf, " AND " );
3590                                 }
3591
3592                                 char* jpred = searchWHERE( filter, right_info, AND_OP_JOIN, NULL );
3593                                 if( jpred ) {
3594                                         OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3595                                         OSRF_BUFFER_ADD( join_buf, jpred );
3596                                         free( jpred );
3597                                 } else {
3598                                         osrfLogError(
3599                                                 OSRF_LOG_MARK,
3600                                                 "%s: JOIN failed.  Invalid conditional expression.",
3601                                                 modulename
3602                                         );
3603                                         jsonIteratorFree( search_itr );
3604                                         buffer_free( join_buf );
3605                                         if( freeable_subhash )
3606                                                 jsonObjectFree( freeable_subhash );
3607                                         if( freeable_hash )
3608                                                 jsonObjectFree( freeable_hash );
3609                                         if( freeable_array )
3610                                                 jsonObjectFree( freeable_array );
3611                                         return NULL;
3612                                 }
3613                         }
3614
3615                         buffer_add( join_buf, " ) " );
3616
3617                         // Recursively add a nested join, if one is present
3618                         const jsonObject* join_filter = jsonObjectGetKeyConst( snode, "join" );
3619                         if( join_filter ) {
3620                                 char* jpred = searchJOIN( join_filter, right_info );
3621                                 if( jpred ) {
3622                                         OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3623                                         OSRF_BUFFER_ADD( join_buf, jpred );
3624                                         free( jpred );
3625                                 } else {
3626                                         osrfLogError( OSRF_LOG_MARK, "%s: Invalid nested join.", modulename );
3627                                         jsonIteratorFree( search_itr );
3628                                         buffer_free( join_buf );
3629                                         if( freeable_subhash )
3630                                                 jsonObjectFree( freeable_subhash );
3631                                         if( freeable_hash )
3632                                                 jsonObjectFree( freeable_hash );
3633                                         if( freeable_array )
3634                                                 jsonObjectFree( freeable_array );
3635                                         return NULL;
3636                                 }
3637                         }
3638                 }
3639
3640                 if( freeable_subhash )
3641                         jsonObjectFree( freeable_subhash );
3642
3643                 jsonIteratorFree( search_itr );
3644         }
3645
3646         if( freeable_hash )
3647                 jsonObjectFree( freeable_hash );
3648
3649         if( freeable_array )
3650                 jsonObjectFree( freeable_array );
3651
3652
3653         return buffer_release( join_buf );
3654 }
3655
3656 /*
3657
3658 { +class : { -or|-and : { field : { op : value }, ... } ... }, ... }
3659 { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }
3660 [ { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }, ... ]
3661
3662 Generate code to express a set of conditions, as for a WHERE clause.  Parameters:
3663
3664 search_hash is the JSON expression of the conditions.
3665 meta is the class definition from the IDL, for the relevant table.
3666 opjoin_type indicates whether multiple conditions, if present, should be
3667         connected by AND or OR.
3668 osrfMethodContext is loaded with all sorts of stuff, but all we do with it here is
3669         to pass it to other functions -- and all they do with it is to use the session
3670         and request members to send error messages back to the client.
3671
3672 */
3673
3674 static char* searchWHERE( const jsonObject* search_hash, const ClassInfo* class_info,
3675                 int opjoin_type, osrfMethodContext* ctx ) {
3676
3677         osrfLogDebug(
3678                 OSRF_LOG_MARK,
3679                 "%s: Entering searchWHERE; search_hash addr = %p, meta addr = %p, "
3680                 "opjoin_type = %d, ctx addr = %p",
3681                 modulename,
3682                 search_hash,
3683                 class_info->class_def,
3684                 opjoin_type,
3685                 ctx
3686         );
3687
3688         growing_buffer* sql_buf = buffer_init( 128 );
3689
3690         jsonObject* node = NULL;
3691
3692         int first = 1;
3693         if( search_hash->type == JSON_ARRAY ) {
3694                 if( 0 == search_hash->size ) {
3695                         osrfLogError(
3696                                 OSRF_LOG_MARK,
3697                                 "%s: Invalid predicate structure: empty JSON array",
3698                                 modulename
3699                         );
3700                         buffer_free( sql_buf );
3701                         return NULL;
3702                 }
3703
3704                 unsigned long i = 0;
3705                 while(( node = jsonObjectGetIndex( search_hash, i++ ) )) {
3706                         if( first ) {
3707                                 first = 0;
3708                         } else {
3709                                 if( opjoin_type == OR_OP_JOIN )
3710                                         buffer_add( sql_buf, " OR " );
3711                                 else
3712                                         buffer_add( sql_buf, " AND " );
3713                         }
3714
3715                         char* subpred = searchWHERE( node, class_info, opjoin_type, ctx );
3716                         if( ! subpred ) {
3717                                 buffer_free( sql_buf );
3718                                 return NULL;
3719                         }
3720
3721                         buffer_fadd( sql_buf, "( %s )", subpred );
3722                         free( subpred );
3723                 }
3724
3725         } else if( search_hash->type == JSON_HASH ) {
3726                 osrfLogDebug( OSRF_LOG_MARK,
3727                         "%s: In WHERE clause, condition type is JSON_HASH", modulename );
3728                 jsonIterator* search_itr = jsonNewIterator( search_hash );
3729                 if( !jsonIteratorHasNext( search_itr ) ) {
3730                         osrfLogError(
3731                                 OSRF_LOG_MARK,
3732                                 "%s: Invalid predicate structure: empty JSON object",
3733                                 modulename
3734                         );
3735                         jsonIteratorFree( search_itr );
3736                         buffer_free( sql_buf );
3737                         return NULL;
3738                 }
3739
3740                 while( (node = jsonIteratorNext( search_itr )) ) {
3741
3742                         if( first ) {
3743                                 first = 0;
3744                         } else {
3745                                 if( opjoin_type == OR_OP_JOIN )
3746                                         buffer_add( sql_buf, " OR " );
3747                                 else
3748                                         buffer_add( sql_buf, " AND " );
3749                         }
3750
3751                         if( '+' == search_itr->key[ 0 ] ) {
3752
3753                                 // This plus sign prefixes a class name or other table alias;
3754                                 // make sure the table alias is in scope
3755                                 ClassInfo* alias_info = search_all_alias( search_itr->key + 1 );
3756                                 if( ! alias_info ) {
3757                                         osrfLogError(
3758                                                          OSRF_LOG_MARK,
3759                                                         "%s: Invalid table alias \"%s\" in WHERE clause",
3760                                                         modulename,
3761                                                         search_itr->key + 1
3762                                         );
3763                                         jsonIteratorFree( search_itr );
3764                                         buffer_free( sql_buf );
3765                                         return NULL;
3766                                 }
3767
3768                                 if( node->type == JSON_STRING ) {
3769                                         // It's the name of a column; make sure it belongs to the class
3770                                         const char* fieldname = jsonObjectGetString( node );
3771                                         if( ! osrfHashGet( alias_info->fields, fieldname ) ) {
3772                                                 osrfLogError(
3773                                                         OSRF_LOG_MARK,
3774                                                         "%s: Invalid column name \"%s\" in WHERE clause "
3775                                                         "for table alias \"%s\"",
3776                                                         modulename,
3777                                                         fieldname,
3778                                                         alias_info->alias
3779                                                 );
3780                                                 jsonIteratorFree( search_itr );
3781                                                 buffer_free( sql_buf );
3782                                                 return NULL;
3783                                         }
3784
3785                                         buffer_fadd( sql_buf, " \"%s\".%s ", alias_info->alias, fieldname );
3786                                 } else {
3787                                         // It's something more complicated
3788                                         char* subpred = searchWHERE( node, alias_info, AND_OP_JOIN, ctx );
3789                                         if( ! subpred ) {
3790                                                 jsonIteratorFree( search_itr );
3791                                                 buffer_free( sql_buf );
3792                                                 return NULL;
3793                                         }
3794
3795                                         buffer_fadd( sql_buf, "( %s )", subpred );
3796                                         free( subpred );
3797                                 }
3798                         } else if( '-' == search_itr->key[ 0 ] ) {
3799                                 if( !strcasecmp( "-or", search_itr->key )) {
3800                                         char* subpred = searchWHERE( node, class_info, OR_OP_JOIN, ctx );
3801                                         if( ! subpred ) {
3802                                                 jsonIteratorFree( search_itr );
3803                                                 buffer_free( sql_buf );
3804                                                 return NULL;
3805                                         }
3806
3807                                         buffer_fadd( sql_buf, "( %s )", subpred );
3808                                         free( subpred );
3809                                 } else if( !strcasecmp( "-and", search_itr->key )) {
3810                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3811                                         if( ! subpred ) {
3812                                                 jsonIteratorFree( search_itr );
3813                                                 buffer_free( sql_buf );
3814                                                 return NULL;
3815                                         }
3816
3817                                         buffer_fadd( sql_buf, "( %s )", subpred );
3818                                         free( subpred );
3819                                 } else if( !strcasecmp("-not",search_itr->key) ) {
3820                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3821                                         if( ! subpred ) {
3822                                                 jsonIteratorFree( search_itr );
3823                                                 buffer_free( sql_buf );
3824                                                 return NULL;
3825                                         }
3826
3827                                         buffer_fadd( sql_buf, " NOT ( %s )", subpred );
3828                                         free( subpred );
3829                                 } else if( !strcasecmp( "-exists", search_itr->key )) {
3830                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3831                                         if( ! subpred ) {
3832                                                 jsonIteratorFree( search_itr );
3833                                                 buffer_free( sql_buf );
3834                                                 return NULL;
3835                                         }
3836
3837                                         buffer_fadd( sql_buf, "EXISTS ( %s )", subpred );
3838                                         free( subpred );
3839                                 } else if( !strcasecmp("-not-exists", search_itr->key )) {
3840                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3841                                         if( ! subpred ) {
3842                                                 jsonIteratorFree( search_itr );
3843                                                 buffer_free( sql_buf );
3844                                                 return NULL;
3845                                         }
3846
3847                                         buffer_fadd( sql_buf, "NOT EXISTS ( %s )", subpred );
3848                                         free( subpred );
3849                                 } else {     // Invalid "minus" operator
3850                                         osrfLogError(
3851                                                          OSRF_LOG_MARK,
3852                                                         "%s: Invalid operator \"%s\" in WHERE clause",
3853                                                         modulename,
3854                                                         search_itr->key
3855                                         );
3856                                         jsonIteratorFree( search_itr );
3857                                         buffer_free( sql_buf );
3858                                         return NULL;
3859                                 }
3860
3861                         } else {
3862
3863                                 const char* class = class_info->class_name;
3864                                 osrfHash* fields = class_info->fields;
3865                                 osrfHash* field = osrfHashGet( fields, search_itr->key );
3866
3867                                 if( !field ) {
3868                                         const char* table = class_info->source_def;
3869                                         osrfLogError(
3870                                                 OSRF_LOG_MARK,
3871                                                 "%s: Attempt to reference non-existent column \"%s\" on %s (%s)",
3872                                                 modulename,
3873                                                 search_itr->key,
3874                                                 table ? table : "?",
3875                                                 class ? class : "?"
3876                                         );
3877                                         jsonIteratorFree( search_itr );
3878                                         buffer_free( sql_buf );
3879                                         return NULL;
3880                                 }
3881
3882                                 char* subpred = searchPredicate( class_info, field, node, ctx );
3883                                 if( ! subpred ) {
3884                                         buffer_free( sql_buf );
3885                                         jsonIteratorFree( search_itr );
3886                                         return NULL;
3887                                 }
3888
3889                                 buffer_add( sql_buf, subpred );
3890                                 free( subpred );
3891                         }
3892                 }
3893                 jsonIteratorFree( search_itr );
3894
3895         } else {
3896                 // ERROR ... only hash and array allowed at this level
3897                 char* predicate_string = jsonObjectToJSON( search_hash );
3898                 osrfLogError(
3899                         OSRF_LOG_MARK,
3900                         "%s: Invalid predicate structure: %s",
3901                         modulename,
3902                         predicate_string
3903                 );
3904                 buffer_free( sql_buf );
3905                 free( predicate_string );
3906                 return NULL;
3907         }
3908
3909         return buffer_release( sql_buf );
3910 }
3911
3912 /* Build a JSON_ARRAY of field names for a given table alias
3913 */
3914 static jsonObject* defaultSelectList( const char* table_alias ) {
3915
3916         if( ! table_alias )
3917                 table_alias = "";
3918
3919         ClassInfo* class_info = search_all_alias( table_alias );
3920         if( ! class_info ) {
3921                 osrfLogError(
3922                         OSRF_LOG_MARK,
3923                         "%s: Can't build default SELECT clause for \"%s\"; no such table alias",
3924                         modulename,
3925                         table_alias
3926                 );
3927                 return NULL;
3928         }
3929
3930         jsonObject* array = jsonNewObjectType( JSON_ARRAY );
3931         osrfHash* field_def = NULL;
3932         osrfHashIterator* field_itr = osrfNewHashIterator( class_info->fields );
3933         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
3934                 const char* field_name = osrfHashIteratorKey( field_itr );
3935                 if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
3936                         jsonObjectPush( array, jsonNewObject( field_name ) );
3937                 }
3938         }
3939         osrfHashIteratorFree( field_itr );
3940
3941         return array;
3942 }
3943
3944 // Translate a jsonObject into a UNION, INTERSECT, or EXCEPT query.
3945 // The jsonObject must be a JSON_HASH with an single entry for "union",
3946 // "intersect", or "except".  The data associated with this key must be an
3947 // array of hashes, each hash being a query.
3948 // Also allowed but currently ignored: entries for "order_by" and "alias".
3949 static char* doCombo( osrfMethodContext* ctx, jsonObject* combo, int flags ) {
3950         // Sanity check
3951         if( ! combo || combo->type != JSON_HASH )
3952                 return NULL;      // should be impossible; validated by caller
3953
3954         const jsonObject* query_array = NULL;   // array of subordinate queries
3955         const char* op = NULL;     // name of operator, e.g. UNION
3956         const char* alias = NULL;  // alias for the query (needed for ORDER BY)
3957         int op_count = 0;          // for detecting conflicting operators
3958         int excepting = 0;         // boolean
3959         int all = 0;               // boolean
3960         jsonObject* order_obj = NULL;
3961
3962         // Identify the elements in the hash
3963         jsonIterator* query_itr = jsonNewIterator( combo );
3964         jsonObject* curr_obj = NULL;
3965         while( (curr_obj = jsonIteratorNext( query_itr ) ) ) {
3966                 if( ! strcmp( "union", query_itr->key ) ) {
3967                         ++op_count;
3968                         op = " UNION ";
3969                         query_array = curr_obj;
3970                 } else if( ! strcmp( "intersect", query_itr->key ) ) {
3971                         ++op_count;
3972                         op = " INTERSECT ";
3973                         query_array = curr_obj;
3974                 } else if( ! strcmp( "except", query_itr->key ) ) {
3975                         ++op_count;
3976                         op = " EXCEPT ";
3977                         excepting = 1;
3978                         query_array = curr_obj;
3979                 } else if( ! strcmp( "order_by", query_itr->key ) ) {
3980                         osrfLogWarning(
3981                                 OSRF_LOG_MARK,
3982                                 "%s: ORDER BY not supported for UNION, INTERSECT, or EXCEPT",
3983                                 modulename
3984                         );
3985                         order_obj = curr_obj;
3986                 } else if( ! strcmp( "alias", query_itr->key ) ) {
3987                         if( curr_obj->type != JSON_STRING ) {
3988                                 jsonIteratorFree( query_itr );
3989                                 return NULL;
3990                         }
3991                         alias = jsonObjectGetString( curr_obj );
3992                 } else if( ! strcmp( "all", query_itr->key ) ) {
3993                         if( obj_is_true( curr_obj ) )
3994                                 all = 1;
3995                 } else {
3996                         if( ctx )
3997                                 osrfAppSessionStatus(
3998                                         ctx->session,
3999                                         OSRF_STATUS_INTERNALSERVERERROR,
4000                                         "osrfMethodException",
4001                                         ctx->request,
4002                                         "Malformed query; unexpected entry in query object"
4003                                 );
4004                         osrfLogError(
4005                                 OSRF_LOG_MARK,
4006                                 "%s: Unexpected entry for \"%s\" in%squery",
4007                                 modulename,
4008                                 query_itr->key,
4009                                 op
4010                         );
4011                         jsonIteratorFree( query_itr );
4012                         return NULL;
4013                 }
4014         }
4015         jsonIteratorFree( query_itr );
4016
4017         // More sanity checks
4018         if( ! query_array ) {
4019                 if( ctx )
4020                         osrfAppSessionStatus(
4021                                 ctx->session,
4022                                 OSRF_STATUS_INTERNALSERVERERROR,
4023                                 "osrfMethodException",
4024                                 ctx->request,
4025                                 "Expected UNION, INTERSECT, or EXCEPT operator not found"
4026                         );
4027                 osrfLogError(
4028                         OSRF_LOG_MARK,
4029                         "%s: Expected UNION, INTERSECT, or EXCEPT operator not found",
4030                         modulename
4031                 );
4032                 return NULL;        // should be impossible...
4033         } else if( op_count > 1 ) {
4034                 if( ctx )
4035                                 osrfAppSessionStatus(
4036                                 ctx->session,
4037                                 OSRF_STATUS_INTERNALSERVERERROR,
4038                                 "osrfMethodException",
4039                                 ctx->request,
4040                                 "Found more than one of UNION, INTERSECT, and EXCEPT in same query"
4041                         );
4042                 osrfLogError(
4043                         OSRF_LOG_MARK,
4044                         "%s: Found more than one of UNION, INTERSECT, and EXCEPT in same query",
4045                         modulename
4046                 );
4047                 return NULL;
4048         } if( query_array->type != JSON_ARRAY ) {
4049                 if( ctx )
4050                                 osrfAppSessionStatus(
4051                                 ctx->session,
4052                                 OSRF_STATUS_INTERNALSERVERERROR,
4053                                 "osrfMethodException",
4054                                 ctx->request,
4055                                 "Malformed query: expected array of queries under UNION, INTERSECT or EXCEPT"
4056                         );
4057                 osrfLogError(
4058                         OSRF_LOG_MARK,
4059                         "%s: Expected JSON_ARRAY of queries for%soperator; found %s",
4060                         modulename,
4061                         op,
4062                         json_type( query_array->type )
4063                 );
4064                 return NULL;
4065         } if( query_array->size < 2 ) {
4066                 if( ctx )
4067                         osrfAppSessionStatus(
4068                                 ctx->session,
4069                                 OSRF_STATUS_INTERNALSERVERERROR,
4070                                 "osrfMethodException",
4071                                 ctx->request,
4072                                 "UNION, INTERSECT or EXCEPT requires multiple queries as operands"
4073                         );
4074                 osrfLogError(
4075                         OSRF_LOG_MARK,
4076                         "%s:%srequires multiple queries as operands",
4077                         modulename,
4078                         op
4079                 );
4080                 return NULL;
4081         } else if( excepting && query_array->size > 2 ) {
4082                 if( ctx )
4083                         osrfAppSessionStatus(
4084                                 ctx->session,
4085                                 OSRF_STATUS_INTERNALSERVERERROR,
4086                                 "osrfMethodException",
4087                                 ctx->request,
4088                                 "EXCEPT operator has too many queries as operands"
4089                         );
4090                 osrfLogError(
4091                         OSRF_LOG_MARK,
4092                         "%s:EXCEPT operator has too many queries as operands",
4093                         modulename
4094                 );
4095                 return NULL;
4096         } else if( order_obj && ! alias ) {
4097                 if( ctx )
4098                         osrfAppSessionStatus(
4099                                 ctx->session,
4100                                 OSRF_STATUS_INTERNALSERVERERROR,
4101                                 "osrfMethodException",
4102                                 ctx->request,
4103                                 "ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT"
4104                         );
4105                 osrfLogError(
4106                         OSRF_LOG_MARK,
4107                         "%s:ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT",
4108                         modulename
4109                 );
4110                 return NULL;
4111         }
4112
4113         // So far so good.  Now build the SQL.
4114         growing_buffer* sql = buffer_init( 256 );
4115
4116         // If we nested inside another UNION, INTERSECT, or EXCEPT,
4117         // Add a layer of parentheses
4118         if( flags & SUBCOMBO )
4119                 OSRF_BUFFER_ADD( sql, "( " );
4120
4121         // Traverse the query array.  Each entry should be a hash.
4122         int first = 1;   // boolean
4123         int i = 0;
4124         jsonObject* query = NULL;
4125         while( (query = jsonObjectGetIndex( query_array, i++ )) ) {
4126                 if( query->type != JSON_HASH ) {
4127                         if( ctx )
4128                                 osrfAppSessionStatus(
4129                                         ctx->session,
4130                                         OSRF_STATUS_INTERNALSERVERERROR,
4131                                         "osrfMethodException",
4132                                         ctx->request,
4133                                         "Malformed query under UNION, INTERSECT or EXCEPT"
4134                                 );
4135                         osrfLogError(
4136                                 OSRF_LOG_MARK,
4137                                 "%s: Malformed query under%s -- expected JSON_HASH, found %s",
4138                                 modulename,
4139                                 op,
4140                                 json_type( query->type )
4141                         );
4142                         buffer_free( sql );
4143                         return NULL;
4144                 }
4145
4146                 if( first )
4147                         first = 0;
4148                 else {
4149                         OSRF_BUFFER_ADD( sql, op );
4150                         if( all )
4151                                 OSRF_BUFFER_ADD( sql, "ALL " );
4152                 }
4153
4154                 char* query_str = buildQuery( ctx, query, SUBSELECT | SUBCOMBO );
4155                 if( ! query_str ) {
4156                         osrfLogError(
4157                                 OSRF_LOG_MARK,
4158                                 "%s: Error building query under%s",
4159                                 modulename,
4160                                 op
4161                         );
4162                         buffer_free( sql );
4163                         return NULL;
4164                 }
4165
4166                 OSRF_BUFFER_ADD( sql, query_str );
4167         }
4168
4169         if( flags & SUBCOMBO )
4170                 OSRF_BUFFER_ADD_CHAR( sql, ')' );
4171
4172         if( !(flags & SUBSELECT) )
4173                 OSRF_BUFFER_ADD_CHAR( sql, ';' );
4174
4175         return buffer_release( sql );
4176 }
4177
4178 // Translate a jsonObject into a SELECT, UNION, INTERSECT, or EXCEPT query.
4179 // The jsonObject must be a JSON_HASH with an entry for "from", "union", "intersect",
4180 // or "except" to indicate the type of query.
4181 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags ) {
4182         // Sanity checks
4183         if( ! query ) {
4184                 if( ctx )
4185                         osrfAppSessionStatus(
4186                                 ctx->session,
4187                                 OSRF_STATUS_INTERNALSERVERERROR,
4188                                 "osrfMethodException",
4189                                 ctx->request,
4190                                 "Malformed query; no query object"
4191                         );
4192                 osrfLogError( OSRF_LOG_MARK, "%s: Null pointer to query object", modulename );
4193                 return NULL;
4194         } else if( query->type != JSON_HASH ) {
4195                 if( ctx )
4196                         osrfAppSessionStatus(
4197                                 ctx->session,
4198                                 OSRF_STATUS_INTERNALSERVERERROR,
4199                                 "osrfMethodException",
4200                                 ctx->request,
4201                                 "Malformed query object"
4202                         );
4203                 osrfLogError(
4204                         OSRF_LOG_MARK,
4205                         "%s: Query object is %s instead of JSON_HASH",
4206                         modulename,
4207                         json_type( query->type )
4208                 );
4209                 return NULL;
4210         }
4211
4212         // Determine what kind of query it purports to be, and dispatch accordingly.
4213         if( jsonObjectGetKeyConst( query, "union" ) ||
4214                 jsonObjectGetKeyConst( query, "intersect" ) ||
4215                 jsonObjectGetKeyConst( query, "except" )) {
4216                 return doCombo( ctx, query, flags );
4217         } else {
4218                 // It is presumably a SELECT query
4219
4220                 // Push a node onto the stack for the current query.  Every level of
4221                 // subquery gets its own QueryFrame on the Stack.
4222                 push_query_frame();
4223
4224                 // Build an SQL SELECT statement
4225                 char* sql = SELECT(
4226                         ctx,
4227                         jsonObjectGetKey( query, "select" ),
4228                         jsonObjectGetKeyConst( query, "from" ),
4229                         jsonObjectGetKeyConst( query, "where" ),
4230                         jsonObjectGetKeyConst( query, "having" ),
4231                         jsonObjectGetKeyConst( query, "order_by" ),
4232                         jsonObjectGetKeyConst( query, "limit" ),
4233                         jsonObjectGetKeyConst( query, "offset" ),
4234                         flags
4235                 );
4236                 pop_query_frame();
4237                 return sql;
4238         }
4239 }
4240
4241 char* SELECT (
4242                 /* method context */ osrfMethodContext* ctx,
4243
4244                 /* SELECT   */ jsonObject* selhash,
4245                 /* FROM     */ const jsonObject* join_hash,
4246                 /* WHERE    */ const jsonObject* search_hash,
4247                 /* HAVING   */ const jsonObject* having_hash,
4248                 /* ORDER BY */ const jsonObject* order_hash,
4249                 /* LIMIT    */ const jsonObject* limit,
4250                 /* OFFSET   */ const jsonObject* offset,
4251                 /* flags    */ int flags
4252 ) {
4253         const char* locale = osrf_message_get_last_locale();
4254
4255         // general tmp objects
4256         const jsonObject* tmp_const;
4257         jsonObject* selclass = NULL;
4258         jsonObject* snode = NULL;
4259         jsonObject* onode = NULL;
4260
4261         char* string = NULL;
4262         int from_function = 0;
4263         int first = 1;
4264         int gfirst = 1;
4265         //int hfirst = 1;
4266
4267         osrfLogDebug(OSRF_LOG_MARK, "cstore SELECT locale: %s", locale ? locale : "(none)" );
4268
4269         // punt if there's no FROM clause
4270         if( !join_hash || ( join_hash->type == JSON_HASH && !join_hash->size )) {
4271                 osrfLogError(
4272                         OSRF_LOG_MARK,
4273                         "%s: FROM clause is missing or empty",
4274                         modulename
4275                 );
4276                 if( ctx )
4277                         osrfAppSessionStatus(
4278                                 ctx->session,
4279                                 OSRF_STATUS_INTERNALSERVERERROR,
4280                                 "osrfMethodException",
4281                                 ctx->request,
4282                                 "FROM clause is missing or empty in JSON query"
4283                         );
4284                 return NULL;
4285         }
4286
4287         // the core search class
4288         const char* core_class = NULL;
4289
4290         // get the core class -- the only key of the top level FROM clause, or a string
4291         if( join_hash->type == JSON_HASH ) {
4292                 jsonIterator* tmp_itr = jsonNewIterator( join_hash );
4293                 snode = jsonIteratorNext( tmp_itr );
4294
4295                 // Populate the current QueryFrame with information
4296                 // about the core class
4297                 if( add_query_core( NULL, tmp_itr->key ) ) {
4298                         if( ctx )
4299                                 osrfAppSessionStatus(
4300                                         ctx->session,
4301                                         OSRF_STATUS_INTERNALSERVERERROR,
4302                                         "osrfMethodException",
4303                                         ctx->request,
4304                                         "Unable to look up core class"
4305                                 );
4306                         return NULL;
4307                 }
4308                 core_class = curr_query->core.class_name;
4309                 join_hash = snode;
4310
4311                 jsonObject* extra = jsonIteratorNext( tmp_itr );
4312
4313                 jsonIteratorFree( tmp_itr );
4314                 snode = NULL;
4315
4316                 // There shouldn't be more than one entry in join_hash
4317                 if( extra ) {
4318                         osrfLogError(
4319                                 OSRF_LOG_MARK,
4320                                 "%s: Malformed FROM clause: extra entry in JSON_HASH",
4321                                 modulename
4322                         );
4323                         if( ctx )
4324                                 osrfAppSessionStatus(
4325                                         ctx->session,
4326                                         OSRF_STATUS_INTERNALSERVERERROR,
4327                                         "osrfMethodException",
4328                                         ctx->request,
4329                                         "Malformed FROM clause in JSON query"
4330                                 );
4331                         return NULL;    // Malformed join_hash; extra entry
4332                 }
4333         } else if( join_hash->type == JSON_ARRAY ) {
4334                 // We're selecting from a function, not from a table
4335                 from_function = 1;
4336                 core_class = jsonObjectGetString( jsonObjectGetIndex( join_hash, 0 ));
4337                 selhash = NULL;
4338
4339         } else if( join_hash->type == JSON_STRING ) {
4340                 // Populate the current QueryFrame with information
4341                 // about the core class
4342                 core_class = jsonObjectGetString( join_hash );
4343                 join_hash = NULL;
4344                 if( add_query_core( NULL, core_class ) ) {
4345                         if( ctx )
4346                                 osrfAppSessionStatus(
4347                                         ctx->session,
4348                                         OSRF_STATUS_INTERNALSERVERERROR,
4349                                         "osrfMethodException",
4350                                         ctx->request,
4351                                         "Unable to look up core class"
4352                                 );
4353                         return NULL;
4354                 }
4355         }
4356         else {
4357                 osrfLogError(
4358                         OSRF_LOG_MARK,
4359                         "%s: FROM clause is unexpected JSON type: %s",
4360                         modulename,
4361                         json_type( join_hash->type )
4362                 );
4363                 if( ctx )
4364                         osrfAppSessionStatus(
4365                                 ctx->session,
4366                                 OSRF_STATUS_INTERNALSERVERERROR,
4367                                 "osrfMethodException",
4368                                 ctx->request,
4369                                 "Ill-formed FROM clause in JSON query"
4370                         );
4371                 return NULL;
4372         }
4373
4374         // Build the join clause, if any, while filling out the list
4375         // of joined classes in the current QueryFrame.
4376         char* join_clause = NULL;
4377         if( join_hash && ! from_function ) {
4378
4379                 join_clause = searchJOIN( join_hash, &curr_query->core );
4380                 if( ! join_clause ) {
4381                         if( ctx )
4382                                 osrfAppSessionStatus(
4383                                         ctx->session,
4384                                         OSRF_STATUS_INTERNALSERVERERROR,
4385                                         "osrfMethodException",
4386                                         ctx->request,
4387                                         "Unable to construct JOIN clause(s)"
4388                                 );
4389                         return NULL;
4390                 }
4391         }
4392
4393         // For in case we don't get a select list
4394         jsonObject* defaultselhash = NULL;
4395
4396         // if there is no select list, build a default select list ...
4397         if( !selhash && !from_function ) {
4398                 jsonObject* default_list = defaultSelectList( core_class );
4399                 if( ! default_list ) {
4400                         if( ctx ) {
4401                                 osrfAppSessionStatus(
4402                                         ctx->session,
4403                                         OSRF_STATUS_INTERNALSERVERERROR,
4404                                         "osrfMethodException",
4405                                         ctx->request,
4406                                         "Unable to build default SELECT clause in JSON query"
4407                                 );
4408                                 free( join_clause );
4409                                 return NULL;
4410                         }
4411                 }
4412
4413                 selhash = defaultselhash = jsonNewObjectType( JSON_HASH );
4414                 jsonObjectSetKey( selhash, core_class, default_list );
4415         }
4416
4417         // The SELECT clause can be encoded only by a hash
4418         if( !from_function && selhash->type != JSON_HASH ) {
4419                 osrfLogError(
4420                         OSRF_LOG_MARK,
4421                         "%s: Expected JSON_HASH for SELECT clause; found %s",
4422                         modulename,
4423                         json_type( selhash->type )
4424                 );
4425
4426                 if( ctx )
4427                         osrfAppSessionStatus(
4428                                 ctx->session,
4429                                 OSRF_STATUS_INTERNALSERVERERROR,
4430                                 "osrfMethodException",
4431                                 ctx->request,
4432                                 "Malformed SELECT clause in JSON query"
4433                         );
4434                 free( join_clause );
4435                 return NULL;
4436         }
4437
4438         // If you see a null or wild card specifier for the core class, or an
4439         // empty array, replace it with a default SELECT list
4440         tmp_const = jsonObjectGetKeyConst( selhash, core_class );
4441         if( tmp_const ) {
4442                 int default_needed = 0;   // boolean
4443                 if( JSON_STRING == tmp_const->type
4444                         && !strcmp( "*", jsonObjectGetString( tmp_const ) ))
4445                                 default_needed = 1;
4446                 else if( JSON_NULL == tmp_const->type )
4447                         default_needed = 1;
4448
4449                 if( default_needed ) {
4450                         // Build a default SELECT list
4451                         jsonObject* default_list = defaultSelectList( core_class );
4452                         if( ! default_list ) {
4453                                 if( ctx ) {
4454                                         osrfAppSessionStatus(
4455                                                 ctx->session,
4456                                                 OSRF_STATUS_INTERNALSERVERERROR,
4457                                                 "osrfMethodException",
4458                                                 ctx->request,
4459                                                 "Can't build default SELECT clause in JSON query"
4460                                         );
4461                                         free( join_clause );
4462                                         return NULL;
4463                                 }
4464                         }
4465
4466                         jsonObjectSetKey( selhash, core_class, default_list );
4467                 }
4468         }
4469
4470         // temp buffers for the SELECT list and GROUP BY clause
4471         growing_buffer* select_buf = buffer_init( 128 );
4472         growing_buffer* group_buf  = buffer_init( 128 );
4473
4474         int aggregate_found = 0;     // boolean
4475
4476         // Build a select list
4477         if( from_function )   // From a function we select everything
4478                 OSRF_BUFFER_ADD_CHAR( select_buf, '*' );
4479         else {
4480
4481                 // Build the SELECT list as SQL
4482             int sel_pos = 1;
4483             first = 1;
4484             gfirst = 1;
4485             jsonIterator* selclass_itr = jsonNewIterator( selhash );
4486             while ( (selclass = jsonIteratorNext( selclass_itr )) ) {    // For each class
4487
4488                         const char* cname = selclass_itr->key;
4489
4490                         // Make sure the target relation is in the FROM clause.
4491
4492                         // At this point join_hash is a step down from the join_hash we
4493                         // received as a parameter.  If the original was a JSON_STRING,
4494                         // then json_hash is now NULL.  If the original was a JSON_HASH,
4495                         // then json_hash is now the first (and only) entry in it,
4496                         // denoting the core class.  We've already excluded the
4497                         // possibility that the original was a JSON_ARRAY, because in
4498                         // that case from_function would be non-NULL, and we wouldn't
4499                         // be here.
4500
4501                         // If the current table alias isn't in scope, bail out
4502                         ClassInfo* class_info = search_alias( cname );
4503                         if( ! class_info ) {
4504                                 osrfLogError(
4505                                         OSRF_LOG_MARK,
4506                                         "%s: SELECT clause references class not in FROM clause: \"%s\"",
4507                                         modulename,
4508                                         cname
4509                                 );
4510                                 if( ctx )
4511                                         osrfAppSessionStatus(
4512                                                 ctx->session,
4513                                                 OSRF_STATUS_INTERNALSERVERERROR,
4514                                                 "osrfMethodException",
4515                                                 ctx->request,
4516                                                 "Selected class not in FROM clause in JSON query"
4517                                         );
4518                                 jsonIteratorFree( selclass_itr );
4519                                 buffer_free( select_buf );
4520                                 buffer_free( group_buf );
4521                                 if( defaultselhash )
4522                                         jsonObjectFree( defaultselhash );
4523                                 free( join_clause );
4524                                 return NULL;
4525                         }
4526
4527                         if( selclass->type != JSON_ARRAY ) {
4528                                 osrfLogError(
4529                                         OSRF_LOG_MARK,
4530                                         "%s: Malformed SELECT list for class \"%s\"; not an array",
4531                                         modulename,
4532                                         cname
4533                                 );
4534                                 if( ctx )
4535                                         osrfAppSessionStatus(
4536                                                 ctx->session,
4537                                                 OSRF_STATUS_INTERNALSERVERERROR,
4538                                                 "osrfMethodException",
4539                                                 ctx->request,
4540                                                 "Selected class not in FROM clause in JSON query"
4541                                         );
4542
4543                                 jsonIteratorFree( selclass_itr );
4544                                 buffer_free( select_buf );
4545                                 buffer_free( group_buf );
4546                                 if( defaultselhash )
4547                                         jsonObjectFree( defaultselhash );
4548                                 free( join_clause );
4549                                 return NULL;
4550                         }
4551
4552                         // Look up some attributes of the current class
4553                         osrfHash* idlClass        = class_info->class_def;
4554                         osrfHash* class_field_set = class_info->fields;
4555                         const char* class_pkey    = osrfHashGet( idlClass, "primarykey" );
4556                         const char* class_tname   = osrfHashGet( idlClass, "tablename" );
4557
4558                         if( 0 == selclass->size ) {
4559                                 osrfLogWarning(
4560                                         OSRF_LOG_MARK,
4561                                         "%s: No columns selected from \"%s\"",
4562                                         modulename,
4563                                         cname
4564                                 );
4565                         }
4566
4567                         // stitch together the column list for the current table alias...
4568                         unsigned long field_idx = 0;
4569                         jsonObject* selfield = NULL;
4570                         while(( selfield = jsonObjectGetIndex( selclass, field_idx++ ) )) {
4571
4572                                 // If we need a separator comma, add one
4573                                 if( first ) {
4574                                         first = 0;
4575                                 } else {
4576                                         OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
4577                                 }
4578
4579                                 // if the field specification is a string, add it to the list
4580                                 if( selfield->type == JSON_STRING ) {
4581
4582                                         // Look up the field in the IDL
4583                                         const char* col_name = jsonObjectGetString( selfield );
4584                                         osrfHash* field_def = NULL;
4585
4586                                         if (!osrfStringArrayContains(
4587                                                         osrfHashGet(
4588                                                                 osrfHashGet( class_field_set, col_name ),
4589                                                                 "suppress_controller"),
4590                                                         modulename
4591                                         ))
4592                                                 field_def = osrfHashGet( class_field_set, col_name );
4593
4594                                         if( !field_def ) {
4595                                                 // No such field in current class
4596                                                 osrfLogError(
4597                                                         OSRF_LOG_MARK,
4598                                                         "%s: Selected column \"%s\" not defined in IDL for class \"%s\"",
4599                                                         modulename,
4600                                                         col_name,
4601                                                         cname
4602                                                 );
4603                                                 if( ctx )
4604                                                         osrfAppSessionStatus(
4605                                                                 ctx->session,
4606                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4607                                                                 "osrfMethodException",
4608                                                                 ctx->request,
4609                                                                 "Selected column not defined in JSON query"
4610                                                         );
4611                                                 jsonIteratorFree( selclass_itr );
4612                                                 buffer_free( select_buf );
4613                                                 buffer_free( group_buf );
4614                                                 if( defaultselhash )
4615                                                         jsonObjectFree( defaultselhash );
4616                                                 free( join_clause );
4617                                                 return NULL;
4618                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
4619                                                 // Virtual field not allowed
4620                                                 osrfLogError(
4621                                                         OSRF_LOG_MARK,
4622                                                         "%s: Selected column \"%s\" for class \"%s\" is virtual",
4623                                                         modulename,
4624                                                         col_name,
4625                                                         cname
4626                                                 );
4627                                                 if( ctx )
4628                                                         osrfAppSessionStatus(
4629                                                                 ctx->session,
4630                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4631                                                                 "osrfMethodException",
4632                                                                 ctx->request,
4633                                                                 "Selected column may not be virtual in JSON query"
4634                                                         );
4635                                                 jsonIteratorFree( selclass_itr );
4636                                                 buffer_free( select_buf );
4637                                                 buffer_free( group_buf );
4638                                                 if( defaultselhash )
4639                                                         jsonObjectFree( defaultselhash );
4640                                                 free( join_clause );
4641                                                 return NULL;
4642                                         }
4643
4644                                         if( locale ) {
4645                                                 const char* i18n;
4646                                                 if( flags & DISABLE_I18N )
4647                                                         i18n = NULL;
4648                                                 else
4649                                                         i18n = osrfHashGet( field_def, "i18n" );
4650
4651                                                 if( str_is_true( i18n ) ) {
4652                                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
4653                                                                 "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
4654                                                                 class_tname, cname, col_name, class_pkey,
4655                                                                 cname, class_pkey, locale, col_name );
4656                                                 } else {
4657                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4658                                                                 cname, col_name, col_name );
4659                                                 }
4660                                         } else {
4661                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4662                                                                 cname, col_name, col_name );
4663                                         }
4664
4665                                 // ... but it could be an object, in which case we check for a Field Transform
4666                                 } else if( selfield->type == JSON_HASH ) {
4667
4668                                         const char* col_name = jsonObjectGetString(
4669                                                         jsonObjectGetKeyConst( selfield, "column" ) );
4670
4671                                         // Get the field definition from the IDL
4672                                         osrfHash* field_def = NULL;
4673                                         if (!osrfStringArrayContains(
4674                                                         osrfHashGet(
4675                                                                 osrfHashGet( class_field_set, col_name ),
4676                                                                 "suppress_controller"),
4677                                                         modulename
4678                                         ))
4679                                                 field_def = osrfHashGet( class_field_set, col_name );
4680
4681
4682                                         if( !field_def ) {
4683                                                 // No such field in current class
4684                                                 osrfLogError(
4685                                                         OSRF_LOG_MARK,
4686                                                         "%s: Selected column \"%s\" is not defined in IDL for class \"%s\"",
4687                                                         modulename,
4688                                                         col_name,
4689                                                         cname
4690                                                 );
4691                                                 if( ctx )
4692                                                         osrfAppSessionStatus(
4693                                                                 ctx->session,
4694                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4695                                                                 "osrfMethodException",
4696                                                                 ctx->request,
4697                                                                 "Selected column is not defined in JSON query"
4698                                                         );
4699                                                 jsonIteratorFree( selclass_itr );
4700                                                 buffer_free( select_buf );
4701                                                 buffer_free( group_buf );
4702                                                 if( defaultselhash )
4703                                                         jsonObjectFree( defaultselhash );
4704                                                 free( join_clause );
4705                                                 return NULL;
4706                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ))) {
4707                                                 // No such field in current class
4708                                                 osrfLogError(
4709                                                         OSRF_LOG_MARK,
4710                                                         "%s: Selected column \"%s\" is virtual for class \"%s\"",
4711                                                         modulename,
4712                                                         col_name,
4713                                                         cname
4714                                                 );
4715                                                 if( ctx )
4716                                                         osrfAppSessionStatus(
4717                                                                 ctx->session,
4718                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4719                                                                 "osrfMethodException",
4720                                                                 ctx->request,
4721                                                                 "Selected column is virtual in JSON query"
4722                                                         );
4723                                                 jsonIteratorFree( selclass_itr );
4724                                                 buffer_free( select_buf );
4725                                                 buffer_free( group_buf );
4726                                                 if( defaultselhash )
4727                                                         jsonObjectFree( defaultselhash );
4728                                                 free( join_clause );
4729                                                 return NULL;
4730                                         }
4731
4732                                         // Decide what to use as a column alias
4733                                         const char* _alias;
4734                                         if((tmp_const = jsonObjectGetKeyConst( selfield, "alias" ))) {
4735                                                 _alias = jsonObjectGetString( tmp_const );
4736                                         } else if((tmp_const = jsonObjectGetKeyConst( selfield, "result_field" ))) { // Use result_field name as the alias
4737                                                 _alias = jsonObjectGetString( tmp_const );
4738                                         } else {         // Use field name as the alias
4739                                                 _alias = col_name;
4740                                         }
4741
4742                                         if( jsonObjectGetKeyConst( selfield, "transform" )) {
4743                                                 char* transform_str = searchFieldTransform(
4744                                                         class_info->alias, field_def, selfield );
4745                                                 if( transform_str ) {
4746                                                         buffer_fadd( select_buf, " %s AS \"%s\"", transform_str, _alias );
4747                                                         free( transform_str );
4748                                                 } else {
4749                                                         if( ctx )
4750                                                                 osrfAppSessionStatus(
4751                                                                         ctx->session,
4752                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4753                                                                         "osrfMethodException",
4754                                                                         ctx->request,
4755                                                                         "Unable to generate transform function in JSON query"
4756                                                                 );
4757                                                         jsonIteratorFree( selclass_itr );
4758                                                         buffer_free( select_buf );
4759                                                         buffer_free( group_buf );
4760                                                         if( defaultselhash )
4761                                                                 jsonObjectFree( defaultselhash );
4762                                                         free( join_clause );
4763                                                         return NULL;
4764                                                 }
4765                                         } else {
4766
4767                                                 if( locale ) {
4768                                                         const char* i18n;
4769                                                         if( flags & DISABLE_I18N )
4770                                                                 i18n = NULL;
4771                                                         else
4772                                                                 i18n = osrfHashGet( field_def, "i18n" );
4773
4774                                                         if( str_is_true( i18n ) ) {
4775                                                                 buffer_fadd( select_buf,
4776                                                                         " oils_i18n_xlate('%s', '%s', '%s', '%s', "
4777                                                                         "\"%s\".%s::TEXT, '%s') AS \"%s\"",
4778                                                                         class_tname, cname, col_name, class_pkey, cname,
4779                                                                         class_pkey, locale, _alias );
4780                                                         } else {
4781                                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4782                                                                         cname, col_name, _alias );
4783                                                         }
4784                                                 } else {
4785                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4786                                                                 cname, col_name, _alias );
4787                                                 }
4788                                         }
4789                                 }
4790                                 else {
4791                                         osrfLogError(
4792                                                 OSRF_LOG_MARK,
4793                                                 "%s: Selected item is unexpected JSON type: %s",
4794                                                 modulename,
4795                                                 json_type( selfield->type )
4796                                         );
4797                                         if( ctx )
4798                                                 osrfAppSessionStatus(
4799                                                         ctx->session,
4800                                                         OSRF_STATUS_INTERNALSERVERERROR,
4801                                                         "osrfMethodException",
4802                                                         ctx->request,
4803                                                         "Ill-formed SELECT item in JSON query"
4804                                                 );
4805                                         jsonIteratorFree( selclass_itr );
4806                                         buffer_free( select_buf );
4807                                         buffer_free( group_buf );
4808                                         if( defaultselhash )
4809                                                 jsonObjectFree( defaultselhash );
4810                                         free( join_clause );
4811                                         return NULL;
4812                                 }
4813
4814                                 const jsonObject* agg_obj = jsonObjectGetKeyConst( selfield, "aggregate" );
4815                                 if( obj_is_true( agg_obj ) )
4816                                         aggregate_found = 1;
4817                                 else {
4818                                         // Append a comma (except for the first one)
4819                                         // and add the column to a GROUP BY clause
4820                                         if( gfirst )
4821                                                 gfirst = 0;
4822                                         else
4823                                                 OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4824
4825                                         buffer_fadd( group_buf, " %d", sel_pos );
4826                                 }
4827
4828 #if 0
4829                             if (is_agg->size || (flags & SELECT_DISTINCT)) {
4830
4831                                         const jsonObject* aggregate_obj = jsonObjectGetKeyConst( selfield, "aggregate");
4832                                     if ( ! obj_is_true( aggregate_obj ) ) {
4833                                             if (gfirst) {
4834                                                     gfirst = 0;
4835                                             } else {
4836                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4837                                             }
4838
4839                                             buffer_fadd(group_buf, " %d", sel_pos);
4840
4841                                         /*
4842                                     } else if (is_agg = jsonObjectGetKeyConst( selfield, "having" )) {
4843                                             if (gfirst) {
4844                                                     gfirst = 0;
4845                                             } else {
4846                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4847                                             }
4848
4849                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4850                                                 OSRF_BUFFER_ADD_CHAR(group_buf, ' ');
4851                                                 OSRF_BUFFER_ADD(group_buf, _column);
4852                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4853                                         */
4854                                     }
4855                             }
4856 #endif
4857
4858                                 sel_pos++;
4859                         } // end while -- iterating across SELECT columns
4860
4861                 } // end while -- iterating across classes
4862
4863                 jsonIteratorFree( selclass_itr );
4864         }
4865
4866         char* col_list = buffer_release( select_buf );
4867
4868         // Make sure the SELECT list isn't empty.  This can happen, for example,
4869         // if we try to build a default SELECT clause from a non-core table.
4870
4871         if( ! *col_list ) {
4872                 osrfLogError( OSRF_LOG_MARK, "%s: SELECT clause is empty", modulename );
4873                 if( ctx )
4874                         osrfAppSessionStatus(
4875                                 ctx->session,
4876                                 OSRF_STATUS_INTERNALSERVERERROR,
4877                                 "osrfMethodException",
4878                                 ctx->request,
4879                                 "SELECT list is empty"
4880                 );
4881                 free( col_list );
4882                 buffer_free( group_buf );
4883                 if( defaultselhash )
4884                         jsonObjectFree( defaultselhash );
4885                 free( join_clause );
4886                 return NULL;
4887         }
4888
4889         char* table = NULL;
4890         if( from_function )
4891                 table = searchValueTransform( join_hash );
4892         else
4893                 table = strdup( curr_query->core.source_def );
4894
4895         if( !table ) {
4896                 if( ctx )
4897                         osrfAppSessionStatus(
4898                                 ctx->session,
4899                                 OSRF_STATUS_INTERNALSERVERERROR,
4900                                 "osrfMethodException",
4901                                 ctx->request,
4902                                 "Unable to identify table for core class"
4903                         );
4904                 free( col_list );
4905                 buffer_free( group_buf );
4906                 if( defaultselhash )
4907                         jsonObjectFree( defaultselhash );
4908                 free( join_clause );
4909                 return NULL;
4910         }
4911
4912         // Put it all together
4913         growing_buffer* sql_buf = buffer_init( 128 );
4914         buffer_fadd(sql_buf, "SELECT %s FROM %s AS \"%s\" ", col_list, table, core_class );
4915         free( col_list );
4916         free( table );
4917
4918         // Append the join clause, if any
4919         if( join_clause ) {
4920                 buffer_add(sql_buf, join_clause );
4921                 free( join_clause );
4922         }
4923
4924         char* order_by_list = NULL;
4925         char* having_buf = NULL;
4926
4927         if( !from_function ) {
4928
4929                 // Build a WHERE clause, if there is one
4930                 if( search_hash ) {
4931                         buffer_add( sql_buf, " WHERE " );
4932
4933                         // and it's on the WHERE clause
4934                         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
4935                         if( ! pred ) {
4936                                 if( ctx ) {
4937                                         osrfAppSessionStatus(
4938                                                 ctx->session,
4939                                                 OSRF_STATUS_INTERNALSERVERERROR,
4940                                                 "osrfMethodException",
4941                                                 ctx->request,
4942                                                 "Severe query error in WHERE predicate -- see error log for more details"
4943                                         );
4944                                 }
4945                                 buffer_free( group_buf );
4946                                 buffer_free( sql_buf );
4947                                 if( defaultselhash )
4948                                         jsonObjectFree( defaultselhash );
4949                                 return NULL;
4950                         }
4951
4952                         buffer_add( sql_buf, pred );
4953                         free( pred );
4954                 }
4955
4956                 // Build a HAVING clause, if there is one
4957                 if( having_hash ) {
4958
4959                         // and it's on the the WHERE clause
4960                         having_buf = searchWHERE( having_hash, &curr_query->core, AND_OP_JOIN, ctx );
4961
4962                         if( ! having_buf ) {
4963                                 if( ctx ) {
4964                                                 osrfAppSessionStatus(
4965                                                 ctx->session,
4966                                                 OSRF_STATUS_INTERNALSERVERERROR,
4967                                                 "osrfMethodException",
4968                                                 ctx->request,
4969                                                 "Severe query error in HAVING predicate -- see error log for more details"
4970                                         );
4971                                 }
4972                                 buffer_free( group_buf );
4973                                 buffer_free( sql_buf );
4974                                 if( defaultselhash )
4975                                         jsonObjectFree( defaultselhash );
4976                                 return NULL;
4977                         }
4978                 }
4979
4980                 // Build an ORDER BY clause, if there is one
4981                 if( NULL == order_hash )
4982                         ;  // No ORDER BY? do nothing
4983                 else if( JSON_ARRAY == order_hash->type ) {
4984                         order_by_list = buildOrderByFromArray( ctx, order_hash );
4985                         if( !order_by_list ) {
4986                                 free( having_buf );
4987                                 buffer_free( group_buf );
4988                                 buffer_free( sql_buf );
4989                                 if( defaultselhash )
4990                                         jsonObjectFree( defaultselhash );
4991                                 return NULL;
4992                         }
4993                 } else if( JSON_HASH == order_hash->type ) {
4994                         // This hash is keyed on class alias.  Each class has either
4995                         // an array of field names or a hash keyed on field name.
4996                         growing_buffer* order_buf = NULL;  // to collect ORDER BY list
4997                         jsonIterator* class_itr = jsonNewIterator( order_hash );
4998                         while( (snode = jsonIteratorNext( class_itr )) ) {
4999
5000                                 ClassInfo* order_class_info = search_alias( class_itr->key );
5001                                 if( ! order_class_info ) {
5002                                         osrfLogError( OSRF_LOG_MARK,
5003                                                 "%s: Invalid class \"%s\" referenced in ORDER BY clause",
5004                                                 modulename, class_itr->key );
5005                                         if( ctx )
5006                                                 osrfAppSessionStatus(
5007                                                         ctx->session,
5008                                                         OSRF_STATUS_INTERNALSERVERERROR,
5009                                                         "osrfMethodException",
5010                                                         ctx->request,
5011                                                         "Invalid class referenced in ORDER BY clause -- "
5012                                                                 "see error log for more details"
5013                                                 );
5014                                         jsonIteratorFree( class_itr );
5015                                         buffer_free( order_buf );
5016                                         free( having_buf );
5017                                         buffer_free( group_buf );
5018                                         buffer_free( sql_buf );
5019                                         if( defaultselhash )
5020                                                 jsonObjectFree( defaultselhash );
5021                                         return NULL;
5022                                 }
5023
5024                                 osrfHash* field_list_def = order_class_info->fields;
5025
5026                                 if( snode->type == JSON_HASH ) {
5027
5028                                         // Hash is keyed on field names from the current class.  For each field
5029                                         // there is another layer of hash to define the sorting details, if any,
5030                                         // or a string to indicate direction of sorting.
5031                                         jsonIterator* order_itr = jsonNewIterator( snode );
5032                                         while( (onode = jsonIteratorNext( order_itr )) ) {
5033
5034                                                 osrfHash* field_def = osrfHashGet( field_list_def, order_itr->key );
5035                                                 if( !field_def ) {
5036                                                         osrfLogError( OSRF_LOG_MARK,
5037                                                                 "%s: Invalid field \"%s\" in ORDER BY clause",
5038                                                                 modulename, order_itr->key );
5039                                                         if( ctx )
5040                                                                 osrfAppSessionStatus(
5041                                                                         ctx->session,
5042                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5043                                                                         "osrfMethodException",
5044                                                                         ctx->request,
5045                                                                         "Invalid field in ORDER BY clause -- "
5046                                                                         "see error log for more details"
5047                                                                 );
5048                                                         jsonIteratorFree( order_itr );
5049                                                         jsonIteratorFree( class_itr );
5050                                                         buffer_free( order_buf );
5051                                                         free( having_buf );
5052                                                         buffer_free( group_buf );
5053                                                         buffer_free( sql_buf );
5054                                                         if( defaultselhash )
5055                                                                 jsonObjectFree( defaultselhash );
5056                                                         return NULL;
5057                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5058                                                         osrfLogError( OSRF_LOG_MARK,
5059                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
5060                                                                 modulename, order_itr->key );
5061                                                         if( ctx )
5062                                                                 osrfAppSessionStatus(
5063                                                                         ctx->session,
5064                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5065                                                                         "osrfMethodException",
5066                                                                         ctx->request,
5067                                                                         "Virtual field in ORDER BY clause -- "
5068                                                                         "see error log for more details"
5069                                                         );
5070                                                         jsonIteratorFree( order_itr );
5071                                                         jsonIteratorFree( class_itr );
5072                                                         buffer_free( order_buf );
5073                                                         free( having_buf );
5074                                                         buffer_free( group_buf );
5075                                                         buffer_free( sql_buf );
5076                                                         if( defaultselhash )
5077                                                                 jsonObjectFree( defaultselhash );
5078                                                         return NULL;
5079                                                 }
5080
5081                                                 const char* direction = NULL;
5082                                                 if( onode->type == JSON_HASH ) {
5083                                                         if( jsonObjectGetKeyConst( onode, "transform" ) ) {
5084                                                                 string = searchFieldTransform(
5085                                                                         class_itr->key,
5086                                                                         osrfHashGet( field_list_def, order_itr->key ),
5087                                                                         onode
5088                                                                 );
5089                                                                 if( ! string ) {
5090                                                                         if( ctx ) osrfAppSessionStatus(
5091                                                                                 ctx->session,
5092                                                                                 OSRF_STATUS_INTERNALSERVERERROR,
5093                                                                                 "osrfMethodException",
5094                                                                                 ctx->request,
5095                                                                                 "Severe query error in ORDER BY clause -- "
5096                                                                                 "see error log for more details"
5097                                                                         );
5098                                                                         jsonIteratorFree( order_itr );
5099                                                                         jsonIteratorFree( class_itr );
5100                                                                         free( having_buf );
5101                                                                         buffer_free( group_buf );
5102                                                                         buffer_free( order_buf);
5103                                                                         buffer_free( sql_buf );
5104                                                                         if( defaultselhash )
5105                                                                                 jsonObjectFree( defaultselhash );
5106                                                                         return NULL;
5107                                                                 }
5108                                                         } else {
5109                                                                 growing_buffer* field_buf = buffer_init( 16 );
5110                                                                 buffer_fadd( field_buf, "\"%s\".%s",
5111                                                                         class_itr->key, order_itr->key );
5112                                                                 string = buffer_release( field_buf );
5113                                                         }
5114
5115                                                         if( (tmp_const = jsonObjectGetKeyConst( onode, "direction" )) ) {
5116                                                                 const char* dir = jsonObjectGetString( tmp_const );
5117                                                                 if(!strncasecmp( dir, "d", 1 )) {
5118                                                                         direction = " DESC";
5119                                                                 } else {
5120                                                                         direction = " ASC";
5121                                                                 }
5122                                                         }
5123
5124                                                 } else if( JSON_NULL == onode->type || JSON_ARRAY == onode->type ) {
5125                                                         osrfLogError( OSRF_LOG_MARK,
5126                                                                 "%s: Expected JSON_STRING in ORDER BY clause; found %s",
5127                                                                 modulename, json_type( onode->type ) );
5128                                                         if( ctx )
5129                                                                 osrfAppSessionStatus(
5130                                                                         ctx->session,
5131                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5132                                                                         "osrfMethodException",
5133                                                                         ctx->request,
5134                                                                         "Malformed ORDER BY clause -- see error log for more details"
5135                                                                 );
5136                                                         jsonIteratorFree( order_itr );
5137                                                         jsonIteratorFree( class_itr );
5138                                                         free( having_buf );
5139                                                         buffer_free( group_buf );
5140                                                         buffer_free( order_buf );
5141                                                         buffer_free( sql_buf );
5142                                                         if( defaultselhash )
5143                                                                 jsonObjectFree( defaultselhash );
5144                                                         return NULL;
5145
5146                                                 } else {
5147                                                         string = strdup( order_itr->key );
5148                                                         const char* dir = jsonObjectGetString( onode );
5149                                                         if( !strncasecmp( dir, "d", 1 )) {
5150                                                                 direction = " DESC";
5151                                                         } else {
5152                                                                 direction = " ASC";
5153                                                         }
5154                                                 }
5155
5156                                                 if( order_buf )
5157                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5158                                                 else
5159                                                         order_buf = buffer_init( 128 );
5160
5161                                                 OSRF_BUFFER_ADD( order_buf, string );
5162                                                 free( string );
5163
5164                                                 if( direction ) {
5165                                                          OSRF_BUFFER_ADD( order_buf, direction );
5166                                                 }
5167
5168                                         } // end while
5169                                         jsonIteratorFree( order_itr );
5170
5171                                 } else if( snode->type == JSON_ARRAY ) {
5172
5173                                         // Array is a list of fields from the current class
5174                                         unsigned long order_idx = 0;
5175                                         while(( onode = jsonObjectGetIndex( snode, order_idx++ ) )) {
5176
5177                                                 const char* _f = jsonObjectGetString( onode );
5178
5179                                                 osrfHash* field_def = osrfHashGet( field_list_def, _f );
5180                                                 if( !field_def ) {
5181                                                         osrfLogError( OSRF_LOG_MARK,
5182                                                                         "%s: Invalid field \"%s\" in ORDER BY clause",
5183                                                                         modulename, _f );
5184                                                         if( ctx )
5185                                                                 osrfAppSessionStatus(
5186                                                                         ctx->session,
5187                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5188                                                                         "osrfMethodException",
5189                                                                         ctx->request,
5190                                                                         "Invalid field in ORDER BY clause -- "
5191                                                                         "see error log for more details"
5192                                                                 );
5193                                                         jsonIteratorFree( class_itr );
5194                                                         buffer_free( order_buf );
5195                                                         free( having_buf );
5196                                                         buffer_free( group_buf );
5197                                                         buffer_free( sql_buf );
5198                                                         if( defaultselhash )
5199                                                                 jsonObjectFree( defaultselhash );
5200                                                         return NULL;
5201                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5202                                                         osrfLogError( OSRF_LOG_MARK,
5203                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
5204                                                                 modulename, _f );
5205                                                         if( ctx )
5206                                                                 osrfAppSessionStatus(
5207                                                                         ctx->session,
5208                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5209                                                                         "osrfMethodException",
5210                                                                         ctx->request,
5211                                                                         "Virtual field in ORDER BY clause -- "
5212                                                                         "see error log for more details"
5213                                                                 );
5214                                                         jsonIteratorFree( class_itr );
5215                                                         buffer_free( order_buf );
5216                                                         free( having_buf );
5217                                                         buffer_free( group_buf );
5218                                                         buffer_free( sql_buf );
5219                                                         if( defaultselhash )
5220                                                                 jsonObjectFree( defaultselhash );
5221                                                         return NULL;
5222                                                 }
5223
5224                                                 if( order_buf )
5225                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5226                                                 else
5227                                                         order_buf = buffer_init( 128 );
5228
5229                                                 buffer_fadd( order_buf, "\"%s\".%s", class_itr->key, _f );
5230
5231                                         } // end while
5232
5233                                 // IT'S THE OOOOOOOOOOOLD STYLE!
5234                                 } else {
5235                                         osrfLogError( OSRF_LOG_MARK,
5236                                                 "%s: Possible SQL injection attempt; direct order by is not allowed",
5237                                                 modulename );
5238                                         if(ctx) {
5239                                                 osrfAppSessionStatus(
5240                                                         ctx->session,
5241                                                         OSRF_STATUS_INTERNALSERVERERROR,
5242                                                         "osrfMethodException",
5243                                                         ctx->request,
5244                                                         "Severe query error -- see error log for more details"
5245                                                 );
5246                                         }
5247
5248                                         free( having_buf );
5249                                         buffer_free( group_buf );
5250                                         buffer_free( order_buf );
5251                                         buffer_free( sql_buf );
5252                                         if( defaultselhash )
5253                                                 jsonObjectFree( defaultselhash );
5254                                         jsonIteratorFree( class_itr );
5255                                         return NULL;
5256                                 }
5257                         } // end while
5258                         jsonIteratorFree( class_itr );
5259                         if( order_buf )
5260                                 order_by_list = buffer_release( order_buf );
5261                 } else {
5262                         osrfLogError( OSRF_LOG_MARK,
5263                                 "%s: Malformed ORDER BY clause; expected JSON_HASH or JSON_ARRAY, found %s",
5264                                 modulename, json_type( order_hash->type ) );
5265                         if( ctx )
5266                                 osrfAppSessionStatus(
5267                                         ctx->session,
5268                                         OSRF_STATUS_INTERNALSERVERERROR,
5269                                         "osrfMethodException",
5270                                         ctx->request,
5271                                         "Malformed ORDER BY clause -- see error log for more details"
5272                                 );
5273                         free( having_buf );
5274                         buffer_free( group_buf );
5275                         buffer_free( sql_buf );
5276                         if( defaultselhash )
5277                                 jsonObjectFree( defaultselhash );
5278                         return NULL;
5279                 }
5280         }
5281
5282         string = buffer_release( group_buf );
5283
5284         if( *string && ( aggregate_found || (flags & SELECT_DISTINCT) ) ) {
5285                 OSRF_BUFFER_ADD( sql_buf, " GROUP BY " );
5286                 OSRF_BUFFER_ADD( sql_buf, string );
5287         }
5288
5289         free( string );
5290
5291         if( having_buf && *having_buf ) {
5292                 OSRF_BUFFER_ADD( sql_buf, " HAVING " );
5293                 OSRF_BUFFER_ADD( sql_buf, having_buf );
5294                 free( having_buf );
5295         }
5296
5297         if( order_by_list ) {
5298
5299                 if( *order_by_list ) {
5300                         OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5301                         OSRF_BUFFER_ADD( sql_buf, order_by_list );
5302                 }
5303
5304                 free( order_by_list );
5305         }
5306
5307         if( limit ){
5308                 const char* str = jsonObjectGetString( limit );
5309                 if (str) { // limit could be JSON_NULL, etc.
5310                         buffer_fadd( sql_buf, " LIMIT %d", atoi( str ));
5311                 }
5312         }
5313
5314         if( offset ) {
5315                 const char* str = jsonObjectGetString( offset );
5316                 if (str) {
5317                         buffer_fadd( sql_buf, " OFFSET %d", atoi( str ));
5318                 }
5319         }
5320
5321         if( !(flags & SUBSELECT) )
5322                 OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5323
5324         if( defaultselhash )
5325                  jsonObjectFree( defaultselhash );
5326
5327         return buffer_release( sql_buf );
5328
5329 } // end of SELECT()
5330
5331 /**
5332         @brief Build a list of ORDER BY expressions.
5333         @param ctx Pointer to the method context.
5334         @param order_array Pointer to a JSON_ARRAY of field specifications.
5335         @return Pointer to a string containing a comma-separated list of ORDER BY expressions.
5336         Each expression may be either a column reference or a function call whose first parameter
5337         is a column reference.
5338
5339         Each entry in @a order_array must be a JSON_HASH with values for "class" and "field".
5340         It may optionally include entries for "direction" and/or "transform".
5341
5342         The calling code is responsible for freeing the returned string.
5343 */
5344 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array ) {
5345         if( ! order_array ) {
5346                 osrfLogError( OSRF_LOG_MARK, "%s: Logic error: NULL pointer for ORDER BY clause",
5347                         modulename );
5348                 if( ctx )
5349                         osrfAppSessionStatus(
5350                                 ctx->session,
5351                                 OSRF_STATUS_INTERNALSERVERERROR,
5352                                 "osrfMethodException",
5353                                 ctx->request,
5354                                 "Logic error: ORDER BY clause expected, not found; "
5355                                         "see error log for more details"
5356                         );
5357                 return NULL;
5358         } else if( order_array->type != JSON_ARRAY ) {
5359                 osrfLogError( OSRF_LOG_MARK,
5360                         "%s: Logic error: Expected JSON_ARRAY for ORDER BY clause, not found", modulename );
5361                 if( ctx )
5362                         osrfAppSessionStatus(
5363                         ctx->session,
5364                         OSRF_STATUS_INTERNALSERVERERROR,
5365                         "osrfMethodException",
5366                         ctx->request,
5367                         "Logic error: Unexpected format for ORDER BY clause; see error log for more details" );
5368                 return NULL;
5369         }
5370
5371         growing_buffer* order_buf = buffer_init( 128 );
5372         int first = 1;        // boolean
5373         int order_idx = 0;
5374         jsonObject* order_spec;
5375         while( (order_spec = jsonObjectGetIndex( order_array, order_idx++ ))) {
5376
5377                 if( JSON_HASH != order_spec->type ) {
5378                         osrfLogError( OSRF_LOG_MARK,
5379                                 "%s: Malformed field specification in ORDER BY clause; "
5380                                 "expected JSON_HASH, found %s",
5381                                 modulename, json_type( order_spec->type ) );
5382                         if( ctx )
5383                                 osrfAppSessionStatus(
5384                                          ctx->session,
5385                                         OSRF_STATUS_INTERNALSERVERERROR,
5386                                         "osrfMethodException",
5387                                         ctx->request,
5388                                         "Malformed ORDER BY clause -- see error log for more details"
5389                                 );
5390                         buffer_free( order_buf );
5391                         return NULL;
5392                 }
5393
5394                 const char* class_alias =
5395                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "class" ));
5396                 const char* field =
5397                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "field" ));
5398
5399                 jsonObject* compare_to = jsonObjectGetKey( order_spec, "compare" );
5400
5401                 if( !field || !class_alias ) {
5402                         osrfLogError( OSRF_LOG_MARK,
5403                                 "%s: Missing class or field name in field specification of ORDER BY clause",
5404                                 modulename );
5405                         if( ctx )
5406                                 osrfAppSessionStatus(
5407                                         ctx->session,
5408                                         OSRF_STATUS_INTERNALSERVERERROR,
5409                                         "osrfMethodException",
5410                                         ctx->request,
5411                                         "Malformed ORDER BY clause -- see error log for more details"
5412                                 );
5413                         buffer_free( order_buf );
5414                         return NULL;
5415                 }
5416
5417                 const ClassInfo* order_class_info = search_alias( class_alias );
5418                 if( ! order_class_info ) {
5419                         osrfLogInternal( OSRF_LOG_MARK, "%s: ORDER BY clause references class \"%s\" "
5420                                 "not in FROM clause, skipping it", modulename, class_alias );
5421                         continue;
5422                 }
5423
5424                 // Add a separating comma, except at the beginning
5425                 if( first )
5426                         first = 0;
5427                 else
5428                         OSRF_BUFFER_ADD( order_buf, ", " );
5429
5430                 osrfHash* field_def = osrfHashGet( order_class_info->fields, field );
5431                 if( !field_def ) {
5432                         osrfLogError( OSRF_LOG_MARK,
5433                                 "%s: Invalid field \"%s\".%s referenced in ORDER BY clause",
5434                                 modulename, class_alias, field );
5435                         if( ctx )
5436                                 osrfAppSessionStatus(
5437                                         ctx->session,
5438                                         OSRF_STATUS_INTERNALSERVERERROR,
5439                                         "osrfMethodException",
5440                                         ctx->request,
5441                                         "Invalid field referenced in ORDER BY clause -- "
5442                                         "see error log for more details"
5443                                 );
5444                         free( order_buf );
5445                         return NULL;
5446                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5447                         osrfLogError( OSRF_LOG_MARK, "%s: Virtual field \"%s\" in ORDER BY clause",
5448                                 modulename, field );
5449                         if( ctx )
5450                                 osrfAppSessionStatus(
5451                                         ctx->session,
5452                                         OSRF_STATUS_INTERNALSERVERERROR,
5453                                         "osrfMethodException",
5454                                         ctx->request,
5455                                         "Virtual field in ORDER BY clause -- see error log for more details"
5456                                 );
5457                         buffer_free( order_buf );
5458                         return NULL;
5459                 }
5460
5461                 if( jsonObjectGetKeyConst( order_spec, "transform" )) {
5462                         char* transform_str = searchFieldTransform( class_alias, field_def, order_spec );
5463                         if( ! transform_str ) {
5464                                 if( ctx )
5465                                         osrfAppSessionStatus(
5466                                                 ctx->session,
5467                                                 OSRF_STATUS_INTERNALSERVERERROR,
5468                                                 "osrfMethodException",
5469                                                 ctx->request,
5470                                                 "Severe query error in ORDER BY clause -- "
5471                                                 "see error log for more details"
5472                                         );
5473                                 buffer_free( order_buf );
5474                                 return NULL;
5475                         }
5476
5477                         OSRF_BUFFER_ADD( order_buf, transform_str );
5478                         free( transform_str );
5479                 } else if( compare_to ) {
5480                         char* compare_str = searchPredicate( order_class_info, field_def, compare_to, ctx );
5481                         if( ! compare_str ) {
5482                                 if( ctx )
5483                                         osrfAppSessionStatus(
5484                                                 ctx->session,
5485                                                 OSRF_STATUS_INTERNALSERVERERROR,
5486                                                 "osrfMethodException",
5487                                                 ctx->request,
5488                                                 "Severe query error in ORDER BY clause -- "
5489                                                 "see error log for more details"
5490                                         );
5491                                 buffer_free( order_buf );
5492                                 return NULL;
5493                         }
5494
5495                         buffer_fadd( order_buf, "(%s)", compare_str );
5496                         free( compare_str );
5497                 }
5498                 else
5499                         buffer_fadd( order_buf, "\"%s\".%s", class_alias, field );
5500
5501                 const char* direction =
5502                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "direction" ) );
5503                 if( direction ) {
5504                         if( direction[ 0 ] && ( 'd' == direction[ 0 ] || 'D' == direction[ 0 ] ) )
5505                                 OSRF_BUFFER_ADD( order_buf, " DESC" );
5506                         else
5507                                 OSRF_BUFFER_ADD( order_buf, " ASC" );
5508                 }
5509         }
5510
5511         return buffer_release( order_buf );
5512 }
5513
5514 /**
5515         @brief Build a SELECT statement.
5516         @param search_hash Pointer to a JSON_HASH or JSON_ARRAY encoding the WHERE clause.
5517         @param rest_of_query Pointer to a JSON_HASH containing any other SQL clauses.
5518         @param meta Pointer to the class metadata for the core class.
5519         @param ctx Pointer to the method context.
5520         @return Pointer to a character string containing the WHERE clause; or NULL upon error.
5521
5522         Within the rest_of_query hash, the meaningful keys are "join", "select", "no_i18n",
5523         "order_by", "limit", and "offset".
5524
5525         The SELECT statements built here are distinct from those built for the json_query method.
5526 */
5527 static char* buildSELECT ( const jsonObject* search_hash, jsonObject* rest_of_query,
5528         osrfHash* meta, osrfMethodContext* ctx ) {
5529
5530         const char* locale = osrf_message_get_last_locale();
5531
5532         osrfHash* fields = osrfHashGet( meta, "fields" );
5533         const char* core_class = osrfHashGet( meta, "classname" );
5534
5535         const jsonObject* join_hash = jsonObjectGetKeyConst( rest_of_query, "join" );
5536
5537         jsonObject* selhash = NULL;
5538         jsonObject* defaultselhash = NULL;
5539
5540         growing_buffer* sql_buf = buffer_init( 128 );
5541         growing_buffer* select_buf = buffer_init( 128 );
5542
5543         if( !(selhash = jsonObjectGetKey( rest_of_query, "select" )) ) {
5544                 defaultselhash = jsonNewObjectType( JSON_HASH );
5545                 selhash = defaultselhash;
5546         }
5547
5548         // If there's no SELECT list for the core class, build one
5549         if( !jsonObjectGetKeyConst( selhash, core_class ) ) {
5550                 jsonObject* field_list = jsonNewObjectType( JSON_ARRAY );
5551
5552                 // Add every non-virtual field to the field list
5553                 osrfHash* field_def = NULL;
5554                 osrfHashIterator* field_itr = osrfNewHashIterator( fields );
5555                 while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
5556                         if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5557                                 const char* field = osrfHashIteratorKey( field_itr );
5558                                 jsonObjectPush( field_list, jsonNewObject( field ) );
5559                         }
5560                 }
5561                 osrfHashIteratorFree( field_itr );
5562                 jsonObjectSetKey( selhash, core_class, field_list );
5563         }
5564
5565         // Build a list of columns for the SELECT clause
5566         int first = 1;
5567         const jsonObject* snode = NULL;
5568         jsonIterator* class_itr = jsonNewIterator( selhash );
5569         while( (snode = jsonIteratorNext( class_itr )) ) {        // For each class
5570
5571                 // If the class isn't in the IDL, ignore it
5572                 const char* cname = class_itr->key;
5573                 osrfHash* idlClass = osrfHashGet( oilsIDL(), cname );
5574                 if( !idlClass )
5575                         continue;
5576
5577                 // If the class isn't the core class, and isn't in the JOIN clause, ignore it
5578                 if( strcmp( core_class, class_itr->key )) {
5579                         if( !join_hash )
5580                                 continue;
5581
5582                         jsonObject* found = jsonObjectFindPath( join_hash, "//%s", class_itr->key );
5583                         if( !found->size ) {
5584                                 jsonObjectFree( found );
5585                                 continue;
5586                         }
5587
5588                         jsonObjectFree( found );
5589                 }
5590
5591                 const jsonObject* node = NULL;
5592                 jsonIterator* select_itr = jsonNewIterator( snode );
5593                 while( (node = jsonIteratorNext( select_itr )) ) {
5594                         const char* item_str = jsonObjectGetString( node );
5595                         osrfHash* field = osrfHashGet( osrfHashGet( idlClass, "fields" ), item_str );
5596                         char* fname = osrfHashGet( field, "name" );
5597
5598                         if( !field )
5599                                 continue;
5600
5601                         if (osrfStringArrayContains( osrfHashGet(field, "suppress_controller"), modulename ))
5602                                 continue;
5603
5604                         if( first ) {
5605                                 first = 0;
5606                         } else {
5607                                 OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
5608                         }
5609
5610                         if( locale ) {
5611                                 const char* i18n;
5612                                 const jsonObject* no_i18n_obj = jsonObjectGetKeyConst( rest_of_query, "no_i18n" );
5613                                 if( obj_is_true( no_i18n_obj ) )    // Suppress internationalization?
5614                                         i18n = NULL;
5615                                 else
5616                                         i18n = osrfHashGet( field, "i18n" );
5617
5618                                 if( str_is_true( i18n ) ) {
5619                                         char* pkey = osrfHashGet( idlClass, "primarykey" );
5620                                         char* tname = osrfHashGet( idlClass, "tablename" );
5621
5622                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
5623                                                         "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
5624                                                         tname, cname, fname, pkey, cname, pkey, locale, fname );
5625                                 } else {
5626                                         buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5627                                 }
5628                         } else {
5629                                 buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5630                         }
5631                 }
5632
5633                 jsonIteratorFree( select_itr );
5634         }
5635
5636         jsonIteratorFree( class_itr );
5637
5638         char* col_list = buffer_release( select_buf );
5639         char* table = oilsGetRelation( meta );
5640         if( !table )
5641                 table = strdup( "(null)" );
5642
5643         buffer_fadd( sql_buf, "SELECT %s FROM %s AS \"%s\"", col_list, table, core_class );
5644         free( col_list );
5645         free( table );
5646
5647         // Clear the query stack (as a fail-safe precaution against possible
5648         // leftover garbage); then push the first query frame onto the stack.
5649         clear_query_stack();
5650         push_query_frame();
5651         if( add_query_core( NULL, core_class ) ) {
5652                 if( ctx )
5653                         osrfAppSessionStatus(
5654                                 ctx->session,
5655                                 OSRF_STATUS_INTERNALSERVERERROR,
5656                                 "osrfMethodException",
5657                                 ctx->request,
5658                                 "Unable to build query frame for core class"
5659                         );
5660                 buffer_free( sql_buf );
5661                 if( defaultselhash )
5662                         jsonObjectFree( defaultselhash );
5663                 return NULL;
5664         }
5665
5666         // Add the JOIN clauses, if any
5667         if( join_hash ) {
5668                 char* join_clause = searchJOIN( join_hash, &curr_query->core );
5669                 OSRF_BUFFER_ADD_CHAR( sql_buf, ' ' );
5670                 OSRF_BUFFER_ADD( sql_buf, join_clause );
5671                 free( join_clause );
5672         }
5673
5674         osrfLogDebug( OSRF_LOG_MARK, "%s pre-predicate SQL =  %s",
5675                 modulename, OSRF_BUFFER_C_STR( sql_buf ));
5676
5677         OSRF_BUFFER_ADD( sql_buf, " WHERE " );
5678
5679         // Add the conditions in the WHERE clause
5680         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
5681         if( !pred ) {
5682                 osrfAppSessionStatus(
5683                         ctx->session,
5684                         OSRF_STATUS_INTERNALSERVERERROR,
5685                                 "osrfMethodException",
5686                                 ctx->request,
5687                                 "Severe query error -- see error log for more details"
5688                         );
5689                 buffer_free( sql_buf );
5690                 if( defaultselhash )
5691                         jsonObjectFree( defaultselhash );
5692                 clear_query_stack();
5693                 return NULL;
5694         } else {
5695                 buffer_add( sql_buf, pred );
5696                 free( pred );
5697         }
5698
5699         // Add the ORDER BY, LIMIT, and/or OFFSET clauses, if present
5700         if( rest_of_query ) {
5701                 const jsonObject* order_by = NULL;
5702                 if( ( order_by = jsonObjectGetKeyConst( rest_of_query, "order_by" )) ){
5703
5704                         char* order_by_list = NULL;
5705
5706                         if( JSON_ARRAY == order_by->type ) {
5707                                 order_by_list = buildOrderByFromArray( ctx, order_by );
5708                                 if( !order_by_list ) {
5709                                         buffer_free( sql_buf );
5710                                         if( defaultselhash )
5711                                                 jsonObjectFree( defaultselhash );
5712                                         clear_query_stack();
5713                                         return NULL;
5714                                 }
5715                         } else if( JSON_HASH == order_by->type ) {
5716                                 // We expect order_by to be a JSON_HASH keyed on class names.  Traverse it
5717                                 // and build a list of ORDER BY expressions.
5718                                 growing_buffer* order_buf = buffer_init( 128 );
5719                                 first = 1;
5720                                 jsonIterator* class_itr = jsonNewIterator( order_by );
5721                                 while( (snode = jsonIteratorNext( class_itr )) ) {  // For each class:
5722
5723                                         ClassInfo* order_class_info = search_alias( class_itr->key );
5724                                         if( ! order_class_info )
5725                                                 continue;    // class not referenced by FROM clause?  Ignore it.
5726
5727                                         if( JSON_HASH == snode->type ) {
5728
5729                                                 // If the data for the current class is a JSON_HASH, then it is
5730                                                 // keyed on field name.
5731
5732                                                 const jsonObject* onode = NULL;
5733                                                 jsonIterator* order_itr = jsonNewIterator( snode );
5734                                                 while( (onode = jsonIteratorNext( order_itr )) ) {  // For each field
5735
5736                                                         osrfHash* field_def = osrfHashGet(
5737                                                                 order_class_info->fields, order_itr->key );
5738                                                         if( !field_def )
5739                                                                 continue;    // Field not defined in IDL?  Ignore it.
5740                                                         if( str_is_true( osrfHashGet( field_def, "virtual")))
5741                                                                 continue;    // Field is virtual?  Ignore it.
5742
5743                                                         char* field_str = NULL;
5744                                                         char* direction = NULL;
5745                                                         if( onode->type == JSON_HASH ) {
5746                                                                 if( jsonObjectGetKeyConst( onode, "transform" ) ) {
5747                                                                         field_str = searchFieldTransform(
5748                                                                                 class_itr->key, field_def, onode );
5749                                                                         if( ! field_str ) {
5750                                                                                 osrfAppSessionStatus(
5751                                                                                         ctx->session,
5752                                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5753                                                                                         "osrfMethodException",
5754                                                                                         ctx->request,
5755                                                                                         "Severe query error in ORDER BY clause -- "
5756                                                                                         "see error log for more details"
5757                                                                                 );
5758                                                                                 jsonIteratorFree( order_itr );
5759                                                                                 jsonIteratorFree( class_itr );
5760                                                                                 buffer_free( order_buf );
5761                                                                                 buffer_free( sql_buf );
5762                                                                                 if( defaultselhash )
5763                                                                                         jsonObjectFree( defaultselhash );
5764                                                                                 clear_query_stack();
5765                                                                                 return NULL;
5766                                                                         }
5767                                                                 } else {
5768                                                                         growing_buffer* field_buf = buffer_init( 16 );
5769                                                                         buffer_fadd( field_buf, "\"%s\".%s",
5770                                                                                 class_itr->key, order_itr->key );
5771                                                                         field_str = buffer_release( field_buf );
5772                                                                 }
5773
5774                                                                 if( ( order_by = jsonObjectGetKeyConst( onode, "direction" )) ) {
5775                                                                         const char* dir = jsonObjectGetString( order_by );
5776                                                                         if(!strncasecmp( dir, "d", 1 )) {
5777                                                                                 direction = " DESC";
5778                                                                         }
5779                                                                 }
5780                                                         } else {
5781                                                                 field_str = strdup( order_itr->key );
5782                                                                 const char* dir = jsonObjectGetString( onode );
5783                                                                 if( !strncasecmp( dir, "d", 1 )) {
5784                                                                         direction = " DESC";
5785                                                                 } else {
5786                                                                         direction = " ASC";
5787                                                                 }
5788                                                         }
5789
5790                                                         if( first ) {
5791                                                                 first = 0;
5792                                                         } else {
5793                                                                 buffer_add( order_buf, ", " );
5794                                                         }
5795
5796                                                         buffer_add( order_buf, field_str );
5797                                                         free( field_str );
5798
5799                                                         if( direction ) {
5800                                                                 buffer_add( order_buf, direction );
5801                                                         }
5802                                                 } // end while; looping over ORDER BY expressions
5803
5804                                                 jsonIteratorFree( order_itr );
5805
5806                                         } else if( JSON_STRING == snode->type ) {
5807                                                 // We expect a comma-separated list of sort fields.
5808                                                 const char* str = jsonObjectGetString( snode );
5809                                                 if( strchr( str, ';' )) {
5810                                                         // No semicolons allowed.  It is theoretically possible for a
5811                                                         // legitimate semicolon to occur within quotes, but it's not likely
5812                                                         // to occur in practice in the context of an ORDER BY list.
5813                                                         osrfLogError( OSRF_LOG_MARK, "%s: Possible attempt at SOL injection -- "
5814                                                                 "semicolon found in ORDER BY list: \"%s\"", modulename, str );
5815                                                         if( ctx ) {
5816                                                                 osrfAppSessionStatus(
5817                                                                         ctx->session,
5818                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5819                                                                         "osrfMethodException",
5820                                                                         ctx->request,
5821                                                                         "Possible attempt at SOL injection -- "
5822                                                                                 "semicolon found in ORDER BY list"
5823                                                                 );
5824                                                         }
5825                                                         jsonIteratorFree( class_itr );
5826                                                         buffer_free( order_buf );
5827                                                         buffer_free( sql_buf );
5828                                                         if( defaultselhash )
5829                                                                 jsonObjectFree( defaultselhash );
5830                                                         clear_query_stack();
5831                                                         return NULL;
5832                                                 }
5833                                                 buffer_add( order_buf, str );
5834                                                 break;
5835                                         }
5836
5837                                 } // end while; looping over order_by classes
5838
5839                                 jsonIteratorFree( class_itr );
5840                                 order_by_list = buffer_release( order_buf );
5841
5842                         } else {
5843                                 osrfLogWarning( OSRF_LOG_MARK,
5844                                         "\"order_by\" object in a query is not a JSON_HASH or JSON_ARRAY;"
5845                                         "no ORDER BY generated" );
5846                         }
5847
5848                         if( order_by_list && *order_by_list ) {
5849                                 OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5850                                 OSRF_BUFFER_ADD( sql_buf, order_by_list );
5851                         }
5852
5853                         free( order_by_list );
5854                 }
5855
5856                 const jsonObject* limit = jsonObjectGetKeyConst( rest_of_query, "limit" );
5857                 if( limit ) {
5858                         const char* str = jsonObjectGetString( limit );
5859                         if (str) {
5860                                 buffer_fadd(
5861                                         sql_buf,
5862                                         " LIMIT %d",
5863                                         atoi(str)
5864                                 );
5865                         }
5866                 }
5867
5868                 const jsonObject* offset = jsonObjectGetKeyConst( rest_of_query, "offset" );
5869                 if( offset ) {
5870                         const char* str = jsonObjectGetString( offset );
5871                         if (str) {
5872                                 buffer_fadd(
5873                                         sql_buf,
5874                                         " OFFSET %d",
5875                                         atoi( str )
5876                                 );
5877                         }
5878                 }
5879         }
5880
5881         if( defaultselhash )
5882                 jsonObjectFree( defaultselhash );
5883         clear_query_stack();
5884
5885         OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5886         return buffer_release( sql_buf );
5887 }
5888
5889 int doJSONSearch ( osrfMethodContext* ctx ) {
5890         if(osrfMethodVerifyContext( ctx )) {
5891                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
5892                 return -1;
5893         }
5894
5895         osrfLogDebug( OSRF_LOG_MARK, "Received query request" );
5896
5897         int err = 0;
5898
5899         jsonObject* hash = jsonObjectGetIndex( ctx->params, 0 );
5900
5901         int flags = 0;
5902
5903         if( obj_is_true( jsonObjectGetKeyConst( hash, "distinct" )))
5904                 flags |= SELECT_DISTINCT;
5905
5906         if( obj_is_true( jsonObjectGetKeyConst( hash, "no_i18n" )))
5907                 flags |= DISABLE_I18N;
5908
5909         osrfLogDebug( OSRF_LOG_MARK, "Building SQL ..." );
5910         clear_query_stack();       // a possibly needless precaution
5911         char* sql = buildQuery( ctx, hash, flags );
5912         clear_query_stack();
5913
5914         if( !sql ) {
5915                 err = -1;
5916                 return err;
5917         }
5918
5919         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
5920
5921         // XXX for now...
5922         dbhandle = writehandle;
5923
5924         dbi_result result = dbi_conn_query( dbhandle, sql );
5925
5926         if( result ) {
5927                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
5928
5929                 if( dbi_result_first_row( result )) {
5930                         /* JSONify the result */
5931                         osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
5932
5933                         do {
5934                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
5935                                 osrfAppRespond( ctx, return_val );
5936                                 jsonObjectFree( return_val );
5937                         } while( dbi_result_next_row( result ));
5938
5939                 } else {
5940                         osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s", modulename, sql );
5941                 }
5942
5943                 osrfAppRespondComplete( ctx, NULL );
5944
5945                 /* clean up the query */
5946                 dbi_result_free( result );
5947
5948         } else {
5949                 err = -1;
5950                 const char* msg;
5951                 int errnum = dbi_conn_error( dbhandle, &msg );
5952                 osrfLogError( OSRF_LOG_MARK, "%s: Error with query [%s]: %d %s",
5953                         modulename, sql, errnum, msg ? msg : "(No description available)" );
5954                 osrfAppSessionStatus(
5955                         ctx->session,
5956                         OSRF_STATUS_INTERNALSERVERERROR,
5957                         "osrfMethodException",
5958                         ctx->request,
5959                         "Severe query error -- see error log for more details"
5960                 );
5961                 if( !oilsIsDBConnected( dbhandle ))
5962                         osrfAppSessionPanic( ctx->session );
5963         }
5964
5965         free( sql );
5966         return err;
5967 }
5968
5969 // The last parameter, err, is used to report an error condition by updating an int owned by
5970 // the calling code.
5971
5972 // In case of an error, we set *err to -1.  If there is no error, *err is left unchanged.
5973 // It is the responsibility of the calling code to initialize *err before the
5974 // call, so that it will be able to make sense of the result.
5975
5976 // Note also that we return NULL if and only if we set *err to -1.  So the err parameter is
5977 // redundant anyway.
5978 static jsonObject* doFieldmapperSearch( osrfMethodContext* ctx, osrfHash* class_meta,
5979                 jsonObject* where_hash, jsonObject* query_hash, int* err ) {
5980
5981         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
5982
5983         // XXX for now...
5984         dbhandle = writehandle;
5985
5986         char* core_class = osrfHashGet( class_meta, "classname" );
5987         osrfLogDebug( OSRF_LOG_MARK, "entering doFieldmapperSearch() with core_class %s", core_class );
5988
5989         char* pkey = osrfHashGet( class_meta, "primarykey" );
5990
5991         if (!ctx->session->userData)
5992                 (void) initSessionCache( ctx );
5993
5994         char *methodtype = osrfHashGet( (osrfHash *) ctx->method->userData, "methodtype" );
5995         char *inside_verify = osrfHashGet( (osrfHash*) ctx->session->userData, "inside_verify" );
5996         int need_to_verify = (inside_verify ? !atoi(inside_verify) : 1);
5997
5998         int i_respond_directly = 0;
5999         int flesh_depth = 0;
6000
6001         char* sql = buildSELECT( where_hash, query_hash, class_meta, ctx );
6002         if( !sql ) {
6003                 osrfLogDebug( OSRF_LOG_MARK, "Problem building query, returning NULL" );
6004                 *err = -1;
6005                 return NULL;
6006         }
6007
6008         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
6009
6010         // Setting the timezone if requested and not in a transaction
6011         if (!getXactId(ctx)) {
6012                 if (tz) {
6013                         setenv("TZ",tz,1);
6014                         tzset();
6015                         dbi_result tz_res = dbi_conn_queryf( writehandle, "SET timezone TO '%s'; -- cstore", tz );
6016                         if( !tz_res ) {
6017                                 osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
6018                                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
6019                                         "osrfMethodException", ctx->request, "Error setting timezone" );
6020                                 if( !oilsIsDBConnected( writehandle )) {
6021                                         osrfAppSessionPanic( ctx->session );
6022                                         return NULL;
6023                                 }
6024                         } else {
6025                                 dbi_result_free( tz_res );
6026                         }
6027                 } else {
6028                         unsetenv("TZ");
6029                         tzset();
6030                         dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- cstore" );
6031                         if( !res ) {
6032                                 osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
6033                                 if( !oilsIsDBConnected( writehandle )) {
6034                                         osrfAppSessionPanic( ctx->session );
6035                                         return NULL;
6036                                 }
6037                         } else {
6038                                 dbi_result_free( res );
6039                         }
6040                 }
6041         }
6042
6043
6044         dbi_result result = dbi_conn_query( dbhandle, sql );
6045
6046         if( NULL == result ) {
6047                 const char* msg;
6048                 int errnum = dbi_conn_error( dbhandle, &msg );
6049                 osrfLogError(OSRF_LOG_MARK, "%s: Error retrieving %s with query [%s]: %d %s",
6050                         modulename, osrfHashGet( class_meta, "fieldmapper" ), sql, errnum,
6051                         msg ? msg : "(No description available)" );
6052                 if( !oilsIsDBConnected( dbhandle ))
6053                         osrfAppSessionPanic( ctx->session );
6054                 osrfAppSessionStatus(
6055                         ctx->session,
6056                         OSRF_STATUS_INTERNALSERVERERROR,
6057                         "osrfMethodException",
6058                         ctx->request,
6059                         "Severe query error -- see error log for more details"
6060                 );
6061                 *err = -1;
6062                 free( sql );
6063                 return NULL;
6064
6065         } else {
6066                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
6067
6068         }
6069
6070         jsonObject* res_list = jsonNewObjectType( JSON_ARRAY );
6071         jsonObject* row_obj = NULL;
6072
6073         // The following two steps are for verifyObjectPCRUD()'s benefit.
6074         // 1. get the flesh depth
6075         const jsonObject* _tmp = jsonObjectGetKeyConst( query_hash, "flesh" );
6076         if( _tmp ) {
6077                 flesh_depth = (int) jsonObjectGetNumber( _tmp );
6078                 if( flesh_depth == -1 || flesh_depth > max_flesh_depth )
6079                         flesh_depth = max_flesh_depth;
6080         }
6081
6082         // 2. figure out one consistent rs_size for verifyObjectPCRUD to use
6083         // over the whole life of this request.  This means if we've already set
6084         // up a rs_size_req_%d, do nothing.
6085         //      a. Incidentally, we can also use this opportunity to set i_respond_directly
6086         int *rs_size = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
6087         if( !rs_size ) {        // pointer null, so value not set in hash
6088                 // i_respond_directly can only be true at the /top/ of a recursive search, if even that.
6089                 i_respond_directly = ( *methodtype == 'r' || *methodtype == 'i' || *methodtype == 's' );
6090
6091                 rs_size = (int *) safe_malloc( sizeof(int) );   // will be freed by sessionDataFree()
6092                 unsigned long long result_count = dbi_result_get_numrows( result );
6093                 *rs_size = (int) result_count * (flesh_depth + 1);      // yes, we could lose some bits, but come on
6094                 osrfHashSet( (osrfHash *) ctx->session->userData, rs_size, "rs_size_req_%d", ctx->request );
6095         }
6096
6097         if( dbi_result_first_row( result )) {
6098
6099                 // Convert each row to a JSON_ARRAY of column values, and enclose those objects
6100                 // in a JSON_ARRAY of rows.  If two or more rows have the same key value, then
6101                 // eliminate the duplicates.
6102                 osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
6103                 osrfHash* dedup = osrfNewHash();
6104                 do {
6105                         row_obj = oilsMakeFieldmapperFromResult( result, class_meta );
6106                         char* pkey_val = oilsFMGetString( row_obj, pkey );
6107                         if( osrfHashGet( dedup, pkey_val ) ) {
6108                                 jsonObjectFree( row_obj );
6109                                 free( pkey_val );
6110                         } else {
6111                                 if( !enforce_pcrud || !need_to_verify ||
6112                                                 verifyObjectPCRUD( ctx, class_meta, row_obj, 0 /* means check user data for rs_size */ )) {
6113                                         osrfHashSet( dedup, pkey_val, pkey_val );
6114                                         jsonObjectPush( res_list, row_obj );
6115                                 }
6116                         }
6117                 } while( dbi_result_next_row( result ));
6118                 osrfHashFree( dedup );
6119
6120         } else {
6121                 osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s",
6122                         modulename, sql );
6123         }
6124
6125         /* clean up the query */
6126         dbi_result_free( result );
6127         free( sql );
6128
6129         // If we're asked to flesh, and there's anything to flesh, then flesh it
6130         // (formerly we would skip fleshing if in pcrud mode, but now we support
6131         // fleshing even in PCRUD).
6132         if( res_list->size ) {
6133                 jsonObject* temp_blob;  // We need a non-zero flesh depth, and a list of fields to flesh
6134                 jsonObject* flesh_fields;
6135                 jsonObject* flesh_blob = NULL;
6136                 osrfStringArray* link_fields = NULL;
6137                 osrfHash* links = NULL;
6138                 int want_flesh = 0;
6139
6140                 if( query_hash ) {
6141                         temp_blob = jsonObjectGetKey( query_hash, "flesh_fields" );
6142                         if( temp_blob && flesh_depth > 0 ) {
6143
6144                                 flesh_blob = jsonObjectClone( temp_blob );
6145                                 flesh_fields = jsonObjectGetKey( flesh_blob, core_class );
6146
6147                                 links = osrfHashGet( class_meta, "links" );
6148
6149                                 // Make an osrfStringArray of the names of fields to be fleshed
6150                                 if( flesh_fields ) {
6151                                         if( flesh_fields->size == 1 ) {
6152                                                 const char* _t = jsonObjectGetString(
6153                                                         jsonObjectGetIndex( flesh_fields, 0 ) );
6154                                                 if( !strcmp( _t, "*" ))
6155                                                         link_fields = osrfHashKeys( links );
6156                                         }
6157
6158                                         if( !link_fields ) {
6159                                                 jsonObject* _f;
6160                                                 link_fields = osrfNewStringArray( 1 );
6161                                                 jsonIterator* _i = jsonNewIterator( flesh_fields );
6162                                                 while ((_f = jsonIteratorNext( _i ))) {
6163                                                         osrfStringArrayAdd( link_fields, jsonObjectGetString( _f ) );
6164                                                 }
6165                                                 jsonIteratorFree( _i );
6166                                         }
6167                                 }
6168                                 want_flesh = link_fields ? 1 : 0;
6169                         }
6170                 }
6171
6172                 osrfHash* fields = osrfHashGet( class_meta, "fields" );
6173
6174                 // Iterate over the JSON_ARRAY of rows
6175                 jsonObject* cur;
6176                 unsigned long res_idx = 0;
6177                 while((cur = jsonObjectGetIndex( res_list, res_idx++ ) )) {
6178
6179                         int i = 0;
6180                         const char* link_field;
6181
6182                         // Iterate over the list of fleshable fields
6183                         if ( want_flesh ) {
6184                                 while( (link_field = osrfStringArrayGetString(link_fields, i++)) ) {
6185
6186                                         osrfLogDebug( OSRF_LOG_MARK, "Starting to flesh %s", link_field );
6187
6188                                         osrfHash* kid_link = osrfHashGet( links, link_field );
6189                                         if( !kid_link )
6190                                                 continue;     // Not a link field; skip it
6191
6192                                         osrfHash* field = osrfHashGet( fields, link_field );
6193                                         if( !field )
6194                                                 continue;     // Not a field at all; skip it (IDL is ill-formed)
6195
6196                                         osrfHash* kid_idl = osrfHashGet( oilsIDL(),
6197                                                 osrfHashGet( kid_link, "class" ));
6198                                         if( !kid_idl )
6199                                                 continue;   // The class it links to doesn't exist; skip it
6200
6201                                         const char* reltype = osrfHashGet( kid_link, "reltype" );
6202                                         if( !reltype )
6203                                                 continue;   // No reltype; skip it (IDL is ill-formed)
6204
6205                                         osrfHash* value_field = field;
6206
6207                                         if(    !strcmp( reltype, "has_many" )
6208                                                 || !strcmp( reltype, "might_have" ) ) { // has_many or might_have
6209                                                 value_field = osrfHashGet(
6210                                                         fields, osrfHashGet( class_meta, "primarykey" ) );
6211                                         }
6212
6213                                         int kid_has_controller = osrfStringArrayContains( osrfHashGet(kid_idl, "controller"), modulename );
6214                                         // fleshing pcrud case: we require the controller in need_to_verify mode
6215                                         if ( !kid_has_controller && enforce_pcrud && need_to_verify ) {
6216                                                 osrfLogInfo( OSRF_LOG_MARK, "%s is not listed as a controller for %s; moving on", modulename, core_class );
6217
6218                                                 jsonObjectSetIndex(
6219                                                         cur,
6220                                                         (unsigned long) atoi( osrfHashGet(field, "array_position") ),
6221                                                         jsonNewObjectType(
6222                                                                 !strcmp( reltype, "has_many" ) ? JSON_ARRAY : JSON_NULL
6223                                                         )
6224                                                 );
6225                                                 continue;
6226                                         }
6227
6228                                         osrfStringArray* link_map = osrfHashGet( kid_link, "map" );
6229
6230                                         if( link_map->size > 0 ) {
6231                                                 jsonObject* _kid_key = jsonNewObjectType( JSON_ARRAY );
6232                                                 jsonObjectPush(
6233                                                         _kid_key,
6234                                                         jsonNewObject( osrfStringArrayGetString( link_map, 0 ) )
6235                                                 );
6236
6237                                                 jsonObjectSetKey(
6238                                                         flesh_blob,
6239                                                         osrfHashGet( kid_link, "class" ),
6240                                                         _kid_key
6241                                                 );
6242                                         };
6243
6244                                         osrfLogDebug(
6245                                                 OSRF_LOG_MARK,
6246                                                 "Link field: %s, remote class: %s, fkey: %s, reltype: %s",
6247                                                 osrfHashGet( kid_link, "field" ),
6248                                                 osrfHashGet( kid_link, "class" ),
6249                                                 osrfHashGet( kid_link, "key" ),
6250                                                 osrfHashGet( kid_link, "reltype" )
6251                                         );
6252
6253                                         const char* search_key = jsonObjectGetString(
6254                                                 jsonObjectGetIndex( cur,
6255                                                         atoi( osrfHashGet( value_field, "array_position" ) )
6256                                                 )
6257                                         );
6258
6259                                         if( !search_key ) {
6260                                                 osrfLogDebug( OSRF_LOG_MARK, "Nothing to search for!" );
6261                                                 continue;
6262                                         }
6263
6264                                         osrfLogDebug( OSRF_LOG_MARK, "Creating param objects..." );
6265
6266                                         // construct WHERE clause
6267                                         jsonObject* where_clause  = jsonNewObjectType( JSON_HASH );
6268                                         jsonObjectSetKey(
6269                                                 where_clause,
6270                                                 osrfHashGet( kid_link, "key" ),
6271                                                 jsonNewObject( search_key )
6272                                         );
6273
6274                                         // construct the rest of the query, mostly
6275                                         // by copying pieces of the previous level of query
6276                                         jsonObject* rest_of_query = jsonNewObjectType( JSON_HASH );
6277                                         jsonObjectSetKey( rest_of_query, "flesh",
6278                                                 jsonNewNumberObject( flesh_depth - 1 + link_map->size )
6279                                         );
6280
6281                                         if( flesh_blob )
6282                                                 jsonObjectSetKey( rest_of_query, "flesh_fields",
6283                                                         jsonObjectClone( flesh_blob ));
6284
6285                                         if( jsonObjectGetKeyConst( query_hash, "order_by" )) {
6286                                                 jsonObjectSetKey( rest_of_query, "order_by",
6287                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "order_by" ))
6288                                                 );
6289                                         }
6290
6291                                         if( jsonObjectGetKeyConst( query_hash, "select" )) {
6292                                                 jsonObjectSetKey( rest_of_query, "select",
6293                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "select" ))
6294                                                 );
6295                                         }
6296
6297                                         // do the query, recursively, to expand the fleshable field
6298                                         jsonObject* kids = doFieldmapperSearch( ctx, kid_idl,
6299                                                 where_clause, rest_of_query, err );
6300
6301                                         jsonObjectFree( where_clause );
6302                                         jsonObjectFree( rest_of_query );
6303
6304                                         if( *err ) {
6305                                                 osrfStringArrayFree( link_fields );
6306                                                 jsonObjectFree( res_list );
6307                                                 jsonObjectFree( flesh_blob );
6308                                                 return NULL;
6309                                         }
6310
6311                                         osrfLogDebug( OSRF_LOG_MARK, "Search for %s return %d linked objects",
6312                                                 osrfHashGet( kid_link, "class" ), kids->size );
6313
6314                                         // Traverse the result set
6315                                         jsonObject* X = NULL;
6316                                         if( link_map->size > 0 && kids->size > 0 ) {
6317                                                 X = kids;
6318                                                 kids = jsonNewObjectType( JSON_ARRAY );
6319
6320                                                 jsonObject* _k_node;
6321                                                 unsigned long res_idx = 0;
6322                                                 while((_k_node = jsonObjectGetIndex( X, res_idx++ ) )) {
6323                                                         jsonObjectPush(
6324                                                                 kids,
6325                                                                 jsonObjectClone(
6326                                                                         jsonObjectGetIndex(
6327                                                                                 _k_node,
6328                                                                                 (unsigned long) atoi(
6329                                                                                         osrfHashGet(
6330                                                                                                 osrfHashGet(
6331                                                                                                         osrfHashGet(
6332                                                                                                                 osrfHashGet(
6333                                                                                                                         oilsIDL(),
6334                                                                                                                         osrfHashGet( kid_link, "class" )
6335                                                                                                                 ),
6336                                                                                                                 "fields"
6337                                                                                                         ),
6338                                                                                                         osrfStringArrayGetString( link_map, 0 )
6339                                                                                                 ),
6340                                                                                                 "array_position"
6341                                                                                         )
6342                                                                                 )
6343                                                                         )
6344                                                                 )
6345                                                         );
6346                                                 } // end while loop traversing X
6347                                         }
6348
6349                                         if (kids->size > 0) {
6350
6351                                                 if((   !strcmp( osrfHashGet( kid_link, "reltype" ), "has_a" )
6352                                                         || !strcmp( osrfHashGet( kid_link, "reltype" ), "might_have" ))
6353                                                 ) {
6354                                                         osrfLogDebug(OSRF_LOG_MARK, "Storing fleshed objects in %s",
6355                                                                 osrfHashGet( kid_link, "field" ));
6356                                                         jsonObjectSetIndex(
6357                                                                 cur,
6358                                                                 (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6359                                                                 jsonObjectClone( jsonObjectGetIndex( kids, 0 ))
6360                                                         );
6361                                                 }
6362                                         }
6363
6364                                         if( !strcmp( osrfHashGet( kid_link, "reltype" ), "has_many" )) {
6365                                                 // has_many
6366                                                 osrfLogDebug( OSRF_LOG_MARK, "Storing fleshed objects in %s",
6367                                                         osrfHashGet( kid_link, "field" ) );
6368                                                 jsonObjectSetIndex(
6369                                                         cur,
6370                                                         (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6371                                                         jsonObjectClone( kids )
6372                                                 );
6373                                         }
6374
6375                                         if( X ) {
6376                                                 jsonObjectFree( kids );
6377                                                 kids = X;
6378                                         }
6379
6380                                         jsonObjectFree( kids );
6381
6382                                         osrfLogDebug( OSRF_LOG_MARK, "Fleshing of %s complete",
6383                                                 osrfHashGet( kid_link, "field" ) );
6384                                         osrfLogDebug( OSRF_LOG_MARK, "%s", jsonObjectToJSON( cur ));
6385
6386                                 } // end while loop traversing list of fleshable fields
6387                         }
6388
6389                         if( i_respond_directly ) {
6390                                 if ( *methodtype == 'i' ) {
6391                                         osrfAppRespond( ctx,
6392                                                 oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
6393                                 } else {
6394                                         osrfAppRespond( ctx, cur );
6395                                 }
6396                         }
6397                 } // end while loop traversing res_list
6398                 jsonObjectFree( flesh_blob );
6399                 osrfStringArrayFree( link_fields );
6400         }
6401
6402         if( i_respond_directly ) {
6403                 jsonObjectFree( res_list );
6404                 return jsonNewObjectType( JSON_ARRAY );
6405         } else {
6406                 return res_list;
6407         }
6408 }
6409
6410
6411 int doUpdate( osrfMethodContext* ctx ) {
6412         if( osrfMethodVerifyContext( ctx )) {
6413                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6414                 return -1;
6415         }
6416
6417         if( enforce_pcrud )
6418                 timeout_needs_resetting = 1;
6419
6420         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6421
6422         jsonObject* target = NULL;
6423         if( enforce_pcrud )
6424                 target = jsonObjectGetIndex( ctx->params, 1 );
6425         else
6426                 target = jsonObjectGetIndex( ctx->params, 0 );
6427
6428         if(!verifyObjectClass( ctx, target )) {
6429                 osrfAppRespondComplete( ctx, NULL );
6430                 return -1;
6431         }
6432
6433         if( getXactId( ctx ) == NULL ) {
6434                 osrfAppSessionStatus(
6435                         ctx->session,
6436                         OSRF_STATUS_BADREQUEST,
6437                         "osrfMethodException",
6438                         ctx->request,
6439                         "No active transaction -- required for UPDATE"
6440                 );
6441                 osrfAppRespondComplete( ctx, NULL );
6442                 return -1;
6443         }
6444
6445         // The following test is harmless but redundant.  If a class is
6446         // readonly, we don't register an update method for it.
6447         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6448                 osrfAppSessionStatus(
6449                         ctx->session,
6450                         OSRF_STATUS_BADREQUEST,
6451                         "osrfMethodException",
6452                         ctx->request,
6453                         "Cannot UPDATE readonly class"
6454                 );
6455                 osrfAppRespondComplete( ctx, NULL );
6456                 return -1;
6457         }
6458
6459         const char* trans_id = getXactId( ctx );
6460
6461         // Set the last_xact_id
6462         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
6463         if( index > -1 ) {
6464                 osrfLogDebug( OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
6465                                 trans_id, target->classname, index );
6466                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
6467         }
6468
6469         char* pkey = osrfHashGet( meta, "primarykey" );
6470         osrfHash* fields = osrfHashGet( meta, "fields" );
6471
6472         char* id = oilsFMGetString( target, pkey );
6473
6474         osrfLogDebug(
6475                 OSRF_LOG_MARK,
6476                 "%s updating %s object with %s = %s",
6477                 modulename,
6478                 osrfHashGet( meta, "fieldmapper" ),
6479                 pkey,
6480                 id
6481         );
6482
6483         dbhandle = writehandle;
6484         growing_buffer* sql = buffer_init( 128 );
6485         buffer_fadd( sql,"UPDATE %s SET", osrfHashGet( meta, "tablename" ));
6486
6487         int first = 1;
6488         osrfHash* field_def = NULL;
6489         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
6490         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
6491
6492                 // Skip virtual fields, and the primary key
6493                 if( str_is_true( osrfHashGet( field_def, "virtual") ) )
6494                         continue;
6495
6496                 if (osrfStringArrayContains( osrfHashGet(field_def, "suppress_controller"), modulename ))
6497                         continue;
6498
6499
6500                 const char* field_name = osrfHashIteratorKey( field_itr );
6501                 if( ! strcmp( field_name, pkey ) )
6502                         continue;
6503
6504                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
6505
6506                 int value_is_numeric = 0;    // boolean
6507                 char* value;
6508                 if( field_object && field_object->classname ) {
6509                         value = oilsFMGetString(
6510                                 field_object,
6511                                 (char*) oilsIDLFindPath( "/%s/primarykey", field_object->classname )
6512                         );
6513                 } else if( field_object && JSON_BOOL == field_object->type ) {
6514                         if( jsonBoolIsTrue( field_object ) )
6515                                 value = strdup( "t" );
6516                         else
6517                                 value = strdup( "f" );
6518                 } else {
6519                         value = jsonObjectToSimpleString( field_object );
6520                         if( field_object && JSON_NUMBER == field_object->type )
6521                                 value_is_numeric = 1;
6522                 }
6523
6524                 osrfLogDebug( OSRF_LOG_MARK, "Updating %s object with %s = %s",
6525                                 osrfHashGet( meta, "fieldmapper" ), field_name, value);
6526
6527                 if( !field_object || field_object->type == JSON_NULL ) {
6528                         if( !( !( strcmp( osrfHashGet( meta, "classname" ), "au" ) )
6529                                         && !( strcmp( field_name, "passwd" ) )) ) { // arg at the special case!
6530                                 if( first )
6531                                         first = 0;
6532                                 else
6533                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6534                                 buffer_fadd( sql, " %s = NULL", field_name );
6535                         }
6536
6537                 } else if( value_is_numeric || !strcmp( get_primitive( field_def ), "number") ) {
6538                         if( first )
6539                                 first = 0;
6540                         else
6541                                 OSRF_BUFFER_ADD_CHAR( sql, ',' );
6542
6543                         const char* numtype = get_datatype( field_def );
6544                         if( !strncmp( numtype, "INT", 3 ) ) {
6545                                 buffer_fadd( sql, " %s = %ld", field_name, atol( value ) );
6546                         } else if( !strcmp( numtype, "NUMERIC" ) ) {
6547                                 buffer_fadd( sql, " %s = %f", field_name, atof( value ) );
6548                         } else {
6549                                 // Must really be intended as a string, so quote it
6550                                 if( dbi_conn_quote_string( dbhandle, &value )) {
6551                                         buffer_fadd( sql, " %s = %s", field_name, value );
6552                                 } else {
6553                                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]",
6554                                                 modulename, value );
6555                                         osrfAppSessionStatus(
6556                                                 ctx->session,
6557                                                 OSRF_STATUS_INTERNALSERVERERROR,
6558                                                 "osrfMethodException",
6559                                                 ctx->request,
6560                                                 "Error quoting string -- please see the error log for more details"
6561                                         );
6562                                         free( value );
6563                                         free( id );
6564                                         osrfHashIteratorFree( field_itr );
6565                                         buffer_free( sql );
6566                                         osrfAppRespondComplete( ctx, NULL );
6567                                         return -1;
6568                                 }
6569                         }
6570
6571                         osrfLogDebug( OSRF_LOG_MARK, "%s is of type %s", field_name, numtype );
6572
6573                 } else {
6574                         if( dbi_conn_quote_string( dbhandle, &value ) ) {
6575                                 if( first )
6576                                         first = 0;
6577                                 else
6578                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6579                                 buffer_fadd( sql, " %s = %s", field_name, value );
6580                         } else {
6581                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
6582                                 osrfAppSessionStatus(
6583                                         ctx->session,
6584                                         OSRF_STATUS_INTERNALSERVERERROR,
6585                                         "osrfMethodException",
6586                                         ctx->request,
6587                                         "Error quoting string -- please see the error log for more details"
6588                                 );
6589                                 free( value );
6590                                 free( id );
6591                                 osrfHashIteratorFree( field_itr );
6592                                 buffer_free( sql );
6593                                 osrfAppRespondComplete( ctx, NULL );
6594                                 return -1;
6595                         }
6596                 }
6597
6598                 free( value );
6599
6600         } // end while
6601
6602         osrfHashIteratorFree( field_itr );
6603
6604         jsonObject* obj = jsonNewObject( id );
6605
6606         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey )), "number" ))
6607                 dbi_conn_quote_string( dbhandle, &id );
6608
6609         buffer_fadd( sql, " WHERE %s = %s;", pkey, id );
6610
6611         char* query = buffer_release( sql );
6612         osrfLogDebug( OSRF_LOG_MARK, "%s: Update SQL [%s]", modulename, query );
6613
6614         dbi_result result = dbi_conn_query( dbhandle, query );
6615         free( query );
6616
6617         int rc = 0;
6618         if( !result ) {
6619                 jsonObjectFree( obj );
6620                 obj = jsonNewObject( NULL );
6621                 const char* msg;
6622                 int errnum = dbi_conn_error( dbhandle, &msg );
6623                 osrfLogError(
6624                         OSRF_LOG_MARK,
6625                         "%s ERROR updating %s object with %s = %s: %d %s",
6626                         modulename,
6627                         osrfHashGet( meta, "fieldmapper" ),
6628                         pkey,
6629                         id,
6630                         errnum,
6631                         msg ? msg : "(No description available)"
6632                 );
6633                 osrfAppSessionStatus(
6634                         ctx->session,
6635                         OSRF_STATUS_INTERNALSERVERERROR,
6636                         "osrfMethodException",
6637                         ctx->request,
6638                         "Error in updating a row -- please see the error log for more details"
6639                 );
6640                 if( !oilsIsDBConnected( dbhandle ))
6641                         osrfAppSessionPanic( ctx->session );
6642                 rc = -1;
6643         } else
6644                 dbi_result_free( result );
6645
6646         free( id );
6647         osrfAppRespondComplete( ctx, obj );
6648         jsonObjectFree( obj );
6649         return rc;
6650 }
6651
6652 int doDelete( osrfMethodContext* ctx ) {
6653         if( osrfMethodVerifyContext( ctx )) {
6654                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6655                 return -1;
6656         }
6657
6658         if( enforce_pcrud )
6659                 timeout_needs_resetting = 1;
6660
6661         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6662
6663         if( getXactId( ctx ) == NULL ) {
6664                 osrfAppSessionStatus(
6665                         ctx->session,
6666                         OSRF_STATUS_BADREQUEST,
6667                         "osrfMethodException",
6668                         ctx->request,
6669                         "No active transaction -- required for DELETE"
6670                 );
6671                 osrfAppRespondComplete( ctx, NULL );
6672                 return -1;
6673         }
6674
6675         // The following test is harmless but redundant.  If a class is
6676         // readonly, we don't register a delete method for it.
6677         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6678                 osrfAppSessionStatus(
6679                         ctx->session,
6680                         OSRF_STATUS_BADREQUEST,
6681                         "osrfMethodException",
6682                         ctx->request,
6683                         "Cannot DELETE readonly class"
6684                 );
6685                 osrfAppRespondComplete( ctx, NULL );
6686                 return -1;
6687         }
6688
6689         dbhandle = writehandle;
6690
6691         char* pkey = osrfHashGet( meta, "primarykey" );
6692
6693         int _obj_pos = 0;
6694         if( enforce_pcrud )
6695                 _obj_pos = 1;
6696
6697         char* id;
6698         if( jsonObjectGetIndex( ctx->params, _obj_pos )->classname ) {
6699                 if( !verifyObjectClass( ctx, jsonObjectGetIndex( ctx->params, _obj_pos ))) {
6700                         osrfAppRespondComplete( ctx, NULL );
6701                         return -1;
6702                 }
6703
6704                 id = oilsFMGetString( jsonObjectGetIndex(ctx->params, _obj_pos), pkey );
6705         } else {
6706                 if( enforce_pcrud && !verifyObjectPCRUD( ctx, meta, NULL, 1 )) {
6707                         osrfAppRespondComplete( ctx, NULL );
6708                         return -1;
6709                 }
6710                 id = jsonObjectToSimpleString( jsonObjectGetIndex( ctx->params, _obj_pos ));
6711         }
6712
6713         osrfLogDebug(
6714                 OSRF_LOG_MARK,
6715                 "%s deleting %s object with %s = %s",
6716                 modulename,
6717                 osrfHashGet( meta, "fieldmapper" ),
6718                 pkey,
6719                 id
6720         );
6721
6722         jsonObject* obj = jsonNewObject( id );
6723
6724         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey ) ), "number" ) )
6725                 dbi_conn_quote_string( writehandle, &id );
6726
6727         dbi_result result = dbi_conn_queryf( writehandle, "DELETE FROM %s WHERE %s = %s;",
6728                 osrfHashGet( meta, "tablename" ), pkey, id );
6729
6730         int rc = 0;
6731         if( !result ) {
6732                 rc = -1;
6733                 jsonObjectFree( obj );
6734                 obj = jsonNewObject( NULL );
6735                 const char* msg;
6736                 int errnum = dbi_conn_error( writehandle, &msg );
6737                 osrfLogError(
6738                         OSRF_LOG_MARK,
6739                         "%s ERROR deleting %s object with %s = %s: %d %s",
6740                         modulename,
6741                         osrfHashGet( meta, "fieldmapper" ),
6742                         pkey,
6743                         id,
6744                         errnum,
6745                         msg ? msg : "(No description available)"
6746                 );
6747                 osrfAppSessionStatus(
6748                         ctx->session,
6749                         OSRF_STATUS_INTERNALSERVERERROR,
6750                         "osrfMethodException",
6751                         ctx->request,
6752                         "Error in deleting a row -- please see the error log for more details"
6753                 );
6754                 if( !oilsIsDBConnected( writehandle ))
6755                         osrfAppSessionPanic( ctx->session );
6756         } else
6757                 dbi_result_free( result );
6758
6759         free( id );
6760
6761         osrfAppRespondComplete( ctx, obj );
6762         jsonObjectFree( obj );
6763         return rc;
6764 }
6765
6766 /**
6767         @brief Translate a row returned from the database into a jsonObject of type JSON_ARRAY.
6768         @param result An iterator for a result set; we only look at the current row.
6769         @param @meta Pointer to the class metadata for the core class.
6770         @return Pointer to the resulting jsonObject if successful; otherwise NULL.
6771
6772         If a column is not defined in the IDL, or if it has no array_position defined for it in
6773         the IDL, or if it is defined as virtual, ignore it.
6774
6775         Otherwise, translate the column value into a jsonObject of type JSON_NULL, JSON_NUMBER,
6776         or JSON_STRING.  Then insert this jsonObject into the JSON_ARRAY according to its
6777         array_position in the IDL.
6778
6779         A field defined in the IDL but not represented in the returned row will leave a hole
6780         in the JSON_ARRAY.  In effect it will be treated as a null value.
6781
6782         In the resulting JSON_ARRAY, the field values appear in the sequence defined by the IDL,
6783         regardless of their sequence in the SELECT statement.  The JSON_ARRAY is assigned the
6784         classname corresponding to the @a meta argument.
6785
6786         The calling code is responsible for freeing the the resulting jsonObject by calling
6787         jsonObjectFree().
6788 */
6789 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result result, osrfHash* meta) {
6790         if( !( result && meta )) return NULL;
6791
6792         jsonObject* object = jsonNewObjectType( JSON_ARRAY );
6793         jsonObjectSetClass( object, osrfHashGet( meta, "classname" ));
6794         osrfLogInternal( OSRF_LOG_MARK, "Setting object class to %s ", object->classname );
6795
6796         osrfHash* fields = osrfHashGet( meta, "fields" );
6797
6798         int columnIndex = 1;
6799         const char* columnName;
6800
6801         /* cycle through the columns in the row returned from the database */
6802         while( (columnName = dbi_result_get_field_name( result, columnIndex )) ) {
6803
6804                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6805
6806                 int fmIndex = -1;  // Will be set to the IDL's sequence number for this field
6807
6808                 /* determine the field type and storage attributes */
6809                 unsigned short type = dbi_result_get_field_type_idx( result, columnIndex );
6810                 int attr            = dbi_result_get_field_attribs_idx( result, columnIndex );
6811
6812                 // Fetch the IDL's sequence number for the field.  If the field isn't in the IDL,
6813                 // or if it has no sequence number there, or if it's virtual, skip it.
6814                 osrfHash* _f = osrfHashGet( fields, (char*) columnName );
6815                 if( _f ) {
6816
6817                         if( str_is_true( osrfHashGet( _f, "virtual" )))
6818                                 continue;   // skip this column: IDL says it's virtual
6819
6820                         const char* pos = (char*) osrfHashGet( _f, "array_position" );
6821                         if( !pos )      // IDL has no sequence number for it.  This shouldn't happen,
6822                                 continue;    // since we assign sequence numbers dynamically as we load the IDL.
6823
6824                         fmIndex = atoi( pos );
6825                         osrfLogInternal( OSRF_LOG_MARK, "... Found column at position [%s]...", pos );
6826                 } else {
6827                         continue;     // This field is not defined in the IDL
6828                 }
6829
6830                 // Stuff the column value into a slot in the JSON_ARRAY, indexed according to the
6831                 // sequence number from the IDL (which is likely to be different from the sequence
6832                 // of columns in the SELECT clause).
6833                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6834                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( NULL ));
6835                 } else {
6836
6837                         switch( type ) {
6838
6839                                 case DBI_TYPE_INTEGER :
6840
6841                                         if( attr & DBI_INTEGER_SIZE8 )
6842                                                 jsonObjectSetIndex( object, fmIndex,
6843                                                         jsonNewNumberObject(
6844                                                                 dbi_result_get_longlong_idx( result, columnIndex )));
6845                                         else
6846                                                 jsonObjectSetIndex( object, fmIndex,
6847                                                         jsonNewNumberObject( dbi_result_get_int_idx( result, columnIndex )));
6848
6849                                         break;
6850
6851                                 case DBI_TYPE_DECIMAL :
6852                                         jsonObjectSetIndex( object, fmIndex,
6853                                                         jsonNewNumberObject( dbi_result_get_double_idx(result, columnIndex )));
6854                                         break;
6855
6856                                 case DBI_TYPE_STRING :
6857
6858                                         jsonObjectSetIndex(
6859                                                 object,
6860                                                 fmIndex,
6861                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex ))
6862                                         );
6863
6864                                         break;
6865
6866                                 case DBI_TYPE_DATETIME : {
6867
6868                                         char dt_string[ 256 ] = "";
6869                                         struct tm gmdt;
6870
6871                                         // Fetch the date column as a time_t
6872                                         time_t _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6873
6874                                         // Translate the time_t to a human-readable string
6875                                         if( !( attr & DBI_DATETIME_DATE )) {
6876                                                 gmtime_r( &_tmp_dt, &gmdt );
6877                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6878                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6879                                                 gmtime_r( &_tmp_dt, &gmdt );
6880                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6881                                         } else {
6882                                                 localtime_r( &_tmp_dt, &gmdt );
6883                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6884                                         }
6885
6886                                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( dt_string ));
6887
6888                                         break;
6889                                 }
6890                                 case DBI_TYPE_BINARY :
6891                                         osrfLogError( OSRF_LOG_MARK,
6892                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6893                         } // End switch
6894                 }
6895                 ++columnIndex;
6896         } // End while
6897
6898         return object;
6899 }
6900
6901 static jsonObject* oilsMakeJSONFromResult( dbi_result result ) {
6902         if( !result ) return NULL;
6903
6904         jsonObject* object = jsonNewObject( NULL );
6905
6906         time_t _tmp_dt;
6907         char dt_string[ 256 ];
6908         struct tm gmdt;
6909
6910         int columnIndex = 1;
6911         int attr;
6912         unsigned short type;
6913         const char* columnName;
6914
6915         /* cycle through the column list */
6916         while(( columnName = dbi_result_get_field_name( result, columnIndex ))) {
6917
6918                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6919
6920                 /* determine the field type and storage attributes */
6921                 type = dbi_result_get_field_type_idx( result, columnIndex );
6922                 attr = dbi_result_get_field_attribs_idx( result, columnIndex );
6923
6924                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6925                         jsonObjectSetKey( object, columnName, jsonNewObject( NULL ));
6926                 } else {
6927
6928                         switch( type ) {
6929
6930                                 case DBI_TYPE_INTEGER :
6931
6932                                         if( attr & DBI_INTEGER_SIZE8 )
6933                                                 jsonObjectSetKey( object, columnName,
6934                                                                 jsonNewNumberObject( dbi_result_get_longlong_idx(
6935                                                                                 result, columnIndex )) );
6936                                         else
6937                                                 jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6938                                                                 dbi_result_get_int_idx( result, columnIndex )) );
6939                                         break;
6940
6941                                 case DBI_TYPE_DECIMAL :
6942                                         jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6943                                                 dbi_result_get_double_idx( result, columnIndex )) );
6944                                         break;
6945
6946                                 case DBI_TYPE_STRING :
6947                                         jsonObjectSetKey( object, columnName,
6948                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex )));
6949                                         break;
6950
6951                                 case DBI_TYPE_DATETIME :
6952
6953                                         memset( dt_string, '\0', sizeof( dt_string ));
6954                                         memset( &gmdt, '\0', sizeof( gmdt ));
6955
6956                                         _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6957
6958                                         if( !( attr & DBI_DATETIME_DATE )) {
6959                                                 gmtime_r( &_tmp_dt, &gmdt );
6960                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6961                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6962                                                 gmtime_r( &_tmp_dt, &gmdt );
6963                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6964                                         } else {
6965                                                 localtime_r( &_tmp_dt, &gmdt );
6966                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6967                                         }
6968
6969                                         jsonObjectSetKey( object, columnName, jsonNewObject( dt_string ));
6970                                         break;
6971
6972                                 case DBI_TYPE_BINARY :
6973                                         osrfLogError( OSRF_LOG_MARK,
6974                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6975                         }
6976                 }
6977                 ++columnIndex;
6978         } // end while loop traversing result
6979
6980         return object;
6981 }
6982
6983 // Interpret a string as true or false
6984 int str_is_true( const char* str ) {
6985         if( NULL == str || strcasecmp( str, "true" ) )
6986                 return 0;
6987         else
6988                 return 1;
6989 }
6990
6991 // Interpret a jsonObject as true or false
6992 static int obj_is_true( const jsonObject* obj ) {
6993         if( !obj )
6994                 return 0;
6995         else switch( obj->type )
6996         {
6997                 case JSON_BOOL :
6998                         if( obj->value.b )
6999                                 return 1;
7000                         else
7001                                 return 0;
7002                 case JSON_STRING :
7003                         if( strcasecmp( obj->value.s, "true" ) )
7004                                 return 0;
7005                         else
7006                                 return 1;
7007                 case JSON_NUMBER :          // Support 1/0 for perl's sake
7008                         if( jsonObjectGetNumber( obj ) == 1.0 )
7009                                 return 1;
7010                         else
7011                                 return 0;
7012                 default :
7013                         return 0;
7014         }
7015 }
7016
7017 // Translate a numeric code into a text string identifying a type of
7018 // jsonObject.  To be used for building error messages.
7019 static const char* json_type( int code ) {
7020         switch ( code )
7021         {
7022                 case 0 :
7023                         return "JSON_HASH";
7024                 case 1 :
7025                         return "JSON_ARRAY";
7026                 case 2 :
7027                         return "JSON_STRING";
7028                 case 3 :
7029                         return "JSON_NUMBER";
7030                 case 4 :
7031                         return "JSON_NULL";
7032                 case 5 :
7033                         return "JSON_BOOL";
7034                 default :
7035                         return "(unrecognized)";
7036         }
7037 }
7038
7039 // Extract the "primitive" attribute from an IDL field definition.
7040 // If we haven't initialized the app, then we must be running in
7041 // some kind of testbed.  In that case, default to "string".
7042 static const char* get_primitive( osrfHash* field ) {
7043         const char* s = osrfHashGet( field, "primitive" );
7044         if( !s ) {
7045                 if( child_initialized )
7046                         osrfLogError(
7047                                 OSRF_LOG_MARK,
7048                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
7049                                 modulename,
7050                                 osrfHashGet( field, "name" )
7051                         );
7052
7053                 s = "string";
7054         }
7055         return s;
7056 }
7057
7058 // Extract the "datatype" attribute from an IDL field definition.
7059 // If we haven't initialized the app, then we must be running in
7060 // some kind of testbed.  In that case, default to to NUMERIC,
7061 // since we look at the datatype only for numbers.
7062 static const char* get_datatype( osrfHash* field ) {
7063         const char* s = osrfHashGet( field, "datatype" );
7064         if( !s ) {
7065                 if( child_initialized )
7066                         osrfLogError(
7067                                 OSRF_LOG_MARK,
7068                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
7069                                 modulename,
7070                                 osrfHashGet( field, "name" )
7071                         );
7072                 else
7073                         s = "NUMERIC";
7074         }
7075         return s;
7076 }
7077
7078 /**
7079         @brief Determine whether a string is potentially a valid SQL identifier.
7080         @param s The identifier to be tested.
7081         @return 1 if the input string is potentially a valid SQL identifier, or 0 if not.
7082
7083         Purpose: to prevent certain kinds of SQL injection.  To that end we don't necessarily
7084         need to follow all the rules exactly, such as requiring that the first character not
7085         be a digit.
7086
7087         We allow leading and trailing white space.  In between, we do not allow punctuation
7088         (except for underscores and dollar signs), control characters, or embedded white space.
7089
7090         More pedantically we should allow quoted identifiers containing arbitrary characters, but
7091         for the foreseeable future such quoted identifiers are not likely to be an issue.
7092 */
7093 int is_identifier( const char* s) {
7094         if( !s )
7095                 return 0;
7096
7097         // Skip leading white space
7098         while( isspace( (unsigned char) *s ) )
7099                 ++s;
7100
7101         if( !s )
7102                 return 0;   // Nothing but white space?  Not okay.
7103
7104         // Check each character until we reach white space or
7105         // end-of-string.  Letters, digits, underscores, and
7106         // dollar signs are okay. With the exception of periods
7107         // (as in schema.identifier), control characters and other
7108         // punctuation characters are not okay.  Anything else
7109         // is okay -- it could for example be part of a multibyte
7110         // UTF8 character such as a letter with diacritical marks,
7111         // and those are allowed.
7112         do {
7113                 if( isalnum( (unsigned char) *s )
7114                         || '.' == *s
7115                         || '_' == *s
7116                         || '$' == *s )
7117                         ;  // Fine; keep going
7118                 else if(   ispunct( (unsigned char) *s )
7119                                 || iscntrl( (unsigned char) *s ) )
7120                         return 0;
7121                 ++s;
7122         } while( *s && ! isspace( (unsigned char) *s ) );
7123
7124         // If we found any white space in the above loop,
7125         // the rest had better be all white space.
7126
7127         while( isspace( (unsigned char) *s ) )
7128                 ++s;
7129
7130         if( *s )
7131                 return 0;   // White space was embedded within non-white space
7132
7133         return 1;
7134 }
7135
7136 /**
7137         @brief Determine whether to accept a character string as a comparison operator.
7138         @param op The candidate comparison operator.
7139         @return 1 if the string is acceptable as a comparison operator, or 0 if not.
7140
7141         We don't validate the operator for real.  We just make sure that it doesn't contain
7142         any semicolons or white space (with special exceptions for a few specific operators).
7143         The idea is to block certain kinds of SQL injection.  If it has no semicolons or white
7144         space but it's still not a valid operator, then the database will complain.
7145
7146         Another approach would be to compare the string against a short list of approved operators.
7147         We don't do that because we want to allow custom operators like ">100*", which at this
7148         writing would be difficult or impossible to express otherwise in a JSON query.
7149 */
7150 int is_good_operator( const char* op ) {
7151         if( !op ) return 0;   // Sanity check
7152
7153         const char* s = op;
7154         while( *s ) {
7155                 if( isspace( (unsigned char) *s ) ) {
7156                         // Special exceptions for SIMILAR TO, IS DISTINCT FROM,
7157                         // and IS NOT DISTINCT FROM.
7158                         if( !strcasecmp( op, "similar to" ) )
7159                                 return 1;
7160                         else if( !strcasecmp( op, "is distinct from" ) )
7161                                 return 1;
7162                         else if( !strcasecmp( op, "is not distinct from" ) )
7163                                 return 1;
7164                         else
7165                                 return 0;
7166                 }
7167                 else if( ';' == *s )
7168                         return 0;
7169                 ++s;
7170         }
7171         return 1;
7172 }
7173
7174 /**
7175         @name Query Frame Management
7176
7177         The following machinery supports a stack of query frames for use by SELECT().
7178
7179         A query frame caches information about one level of a SELECT query.  When we enter
7180         a subquery, we push another query frame onto the stack, and pop it off when we leave.
7181
7182         The query frame stores information about the core class, and about any joined classes
7183         in the FROM clause.
7184
7185         The main purpose is to map table aliases to classes and tables, so that a query can
7186         join to the same table more than once.  A secondary goal is to reduce the number of
7187         lookups in the IDL by caching the results.
7188 */
7189 /*@{*/
7190
7191 #define STATIC_CLASS_INFO_COUNT 3
7192
7193 static ClassInfo static_class_info[ STATIC_CLASS_INFO_COUNT ];
7194
7195 /**
7196         @brief Allocate a ClassInfo as raw memory.
7197         @return Pointer to the newly allocated ClassInfo.
7198
7199         Except for the in_use flag, which is used only by the allocation and deallocation
7200         logic, we don't initialize the ClassInfo here.
7201 */
7202 static ClassInfo* allocate_class_info( void ) {
7203         // In order to reduce the number of mallocs and frees, we return a static
7204         // instance of ClassInfo, if we can find one that we're not already using.
7205         // We rely on the fact that the compiler will implicitly initialize the
7206         // static instances so that in_use == 0.
7207
7208         int i;
7209         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7210                 if( ! static_class_info[ i ].in_use ) {
7211                         static_class_info[ i ].in_use = 1;
7212                         return static_class_info + i;
7213                 }
7214         }
7215
7216         // The static ones are all in use.  Malloc one.
7217
7218         return safe_malloc( sizeof( ClassInfo ) );
7219 }
7220
7221 /**
7222         @brief Free any malloc'd memory owned by a ClassInfo, returning it to a pristine state.
7223         @param info Pointer to the ClassInfo to be cleared.
7224 */
7225 static void clear_class_info( ClassInfo* info ) {
7226         // Sanity check
7227         if( ! info )
7228                 return;
7229
7230         // Free any malloc'd strings
7231
7232         if( info->alias != info->alias_store )
7233                 free( info->alias );
7234
7235         if( info->class_name != info->class_name_store )
7236                 free( info->class_name );
7237
7238         free( info->source_def );
7239
7240         info->alias = info->class_name = info->source_def = NULL;
7241         info->next = NULL;
7242 }
7243
7244 /**
7245         @brief Free a ClassInfo and everything it owns.
7246         @param info Pointer to the ClassInfo to be freed.
7247 */
7248 static void free_class_info( ClassInfo* info ) {
7249         // Sanity check
7250         if( ! info )
7251                 return;
7252
7253         clear_class_info( info );
7254
7255         // If it's one of the static instances, just mark it as not in use
7256
7257         int i;
7258         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7259                 if( info == static_class_info + i ) {
7260                         static_class_info[ i ].in_use = 0;
7261                         return;
7262                 }
7263         }
7264
7265         // Otherwise it must have been malloc'd, so free it
7266
7267         free( info );
7268 }
7269
7270 /**
7271         @brief Populate an already-allocated ClassInfo.
7272         @param info Pointer to the ClassInfo to be populated.
7273         @param alias Alias for the class.  If it is NULL, or an empty string, use the class
7274         name for an alias.
7275         @param class Name of the class.
7276         @return Zero if successful, or 1 if not.
7277
7278         Populate the ClassInfo with copies of the alias and class name, and with pointers to
7279         the relevant portions of the IDL for the specified class.
7280 */
7281 static int build_class_info( ClassInfo* info, const char* alias, const char* class ) {
7282         // Sanity checks
7283         if( ! info ){
7284                 osrfLogError( OSRF_LOG_MARK,
7285                                           "%s ERROR: No ClassInfo available to populate", modulename );
7286                 info->alias = info->class_name = info->source_def = NULL;
7287                 info->class_def = info->fields = info->links = NULL;
7288                 return 1;
7289         }
7290
7291         if( ! class ) {
7292                 osrfLogError( OSRF_LOG_MARK,
7293                                           "%s ERROR: No class name provided for lookup", modulename );
7294                 info->alias = info->class_name = info->source_def = NULL;
7295                 info->class_def = info->fields = info->links = NULL;
7296                 return 1;
7297         }
7298
7299         // Alias defaults to class name if not supplied
7300         if( ! alias || ! alias[ 0 ] )
7301                 alias = class;
7302
7303         // Look up class info in the IDL
7304         osrfHash* class_def = osrfHashGet( oilsIDL(), class );
7305         if( ! class_def ) {
7306                 osrfLogError( OSRF_LOG_MARK,
7307                                           "%s ERROR: Class %s not defined in IDL", modulename, class );
7308                 info->alias = info->class_name = info->source_def = NULL;
7309                 info->class_def = info->fields = info->links = NULL;
7310                 return 1;
7311         } else if( str_is_true( osrfHashGet( class_def, "virtual" ) ) ) {
7312                 osrfLogError( OSRF_LOG_MARK,
7313                                           "%s ERROR: Class %s is defined as virtual", modulename, class );
7314                 info->alias = info->class_name = info->source_def = NULL;
7315                 info->class_def = info->fields = info->links = NULL;
7316                 return 1;
7317         }
7318
7319         osrfHash* links = osrfHashGet( class_def, "links" );
7320         if( ! links ) {
7321                 osrfLogError( OSRF_LOG_MARK,
7322                                           "%s ERROR: No links defined in IDL for class %s", modulename, class );
7323                 info->alias = info->class_name = info->source_def = NULL;
7324                 info->class_def = info->fields = info->links = NULL;
7325                 return 1;
7326         }
7327
7328         osrfHash* fields = osrfHashGet( class_def, "fields" );
7329         if( ! fields ) {
7330                 osrfLogError( OSRF_LOG_MARK,
7331                                           "%s ERROR: No fields defined in IDL for class %s", modulename, class );
7332                 info->alias = info->class_name = info->source_def = NULL;
7333                 info->class_def = info->fields = info->links = NULL;
7334                 return 1;
7335         }
7336
7337         char* source_def = oilsGetRelation( class_def );
7338         if( ! source_def )
7339                 return 1;
7340
7341         // We got everything we need, so populate the ClassInfo
7342         if( strlen( alias ) > ALIAS_STORE_SIZE )
7343                 info->alias = strdup( alias );
7344         else {
7345                 strcpy( info->alias_store, alias );
7346                 info->alias = info->alias_store;
7347         }
7348
7349         if( strlen( class ) > CLASS_NAME_STORE_SIZE )
7350                 info->class_name = strdup( class );
7351         else {
7352                 strcpy( info->class_name_store, class );
7353                 info->class_name = info->class_name_store;
7354         }
7355
7356         info->source_def = source_def;
7357
7358         info->class_def = class_def;
7359         info->links     = links;
7360         info->fields    = fields;
7361
7362         return 0;
7363 }
7364
7365 #define STATIC_FRAME_COUNT 3
7366
7367 static QueryFrame static_frame[ STATIC_FRAME_COUNT ];
7368
7369 /**
7370         @brief Allocate a QueryFrame as raw memory.
7371         @return Pointer to the newly allocated QueryFrame.
7372
7373         Except for the in_use flag, which is used only by the allocation and deallocation
7374         logic, we don't initialize the QueryFrame here.
7375 */
7376 static QueryFrame* allocate_frame( void ) {
7377         // In order to reduce the number of mallocs and frees, we return a static
7378         // instance of QueryFrame, if we can find one that we're not already using.
7379         // We rely on the fact that the compiler will implicitly initialize the
7380         // static instances so that in_use == 0.
7381
7382         int i;
7383         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7384                 if( ! static_frame[ i ].in_use ) {
7385                         static_frame[ i ].in_use = 1;
7386                         return static_frame + i;
7387                 }
7388         }
7389
7390         // The static ones are all in use.  Malloc one.
7391
7392         return safe_malloc( sizeof( QueryFrame ) );
7393 }
7394
7395 /**
7396         @brief Free a QueryFrame, and all the memory it owns.
7397         @param frame Pointer to the QueryFrame to be freed.
7398 */
7399 static void free_query_frame( QueryFrame* frame ) {
7400         // Sanity check
7401         if( ! frame )
7402                 return;
7403
7404         clear_class_info( &frame->core );
7405
7406         // Free the join list
7407         ClassInfo* temp;
7408         ClassInfo* info = frame->join_list;
7409         while( info ) {
7410                 temp = info->next;
7411                 free_class_info( info );
7412                 info = temp;
7413         }
7414
7415         frame->join_list = NULL;
7416         frame->next = NULL;
7417
7418         // If the frame is a static instance, just mark it as unused
7419         int i;
7420         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7421                 if( frame == static_frame + i ) {
7422                         static_frame[ i ].in_use = 0;
7423                         return;
7424                 }
7425         }
7426
7427         // Otherwise it must have been malloc'd, so free it
7428
7429         free( frame );
7430 }
7431
7432 /**
7433         @brief Search a given QueryFrame for a specified alias.
7434         @param frame Pointer to the QueryFrame to be searched.
7435         @param target The alias for which to search.
7436         @return Pointer to the ClassInfo for the specified alias, if found; otherwise NULL.
7437 */
7438 static ClassInfo* search_alias_in_frame( QueryFrame* frame, const char* target ) {
7439         if( ! frame || ! target ) {
7440                 return NULL;
7441         }
7442
7443         ClassInfo* found_class = NULL;
7444
7445         if( !strcmp( target, frame->core.alias ) )
7446                 return &(frame->core);
7447         else {
7448                 ClassInfo* curr_class = frame->join_list;
7449                 while( curr_class ) {
7450                         if( strcmp( target, curr_class->alias ) )
7451                                 curr_class = curr_class->next;
7452                         else {
7453                                 found_class = curr_class;
7454                                 break;
7455                         }
7456                 }
7457         }
7458
7459         return found_class;
7460 }
7461
7462 /**
7463         @brief Push a new (blank) QueryFrame onto the stack.
7464 */
7465 static void push_query_frame( void ) {
7466         QueryFrame* frame = allocate_frame();
7467         frame->join_list = NULL;
7468         frame->next = curr_query;
7469
7470         // Initialize the ClassInfo for the core class
7471         ClassInfo* core = &frame->core;
7472         core->alias = core->class_name = core->source_def = NULL;
7473         core->class_def = core->fields = core->links = NULL;
7474
7475         curr_query = frame;
7476 }
7477
7478 /**
7479         @brief Pop a QueryFrame off the stack and destroy it.
7480 */
7481 static void pop_query_frame( void ) {
7482         // Sanity check
7483         if( ! curr_query )
7484                 return;
7485
7486         QueryFrame* popped = curr_query;
7487         curr_query = popped->next;
7488
7489         free_query_frame( popped );
7490 }
7491
7492 /**
7493         @brief Populate the ClassInfo for the core class.
7494         @param alias Alias for the core class.  If it is NULL or an empty string, we use the
7495         class name as an alias.
7496         @param class_name Name of the core class.
7497         @return Zero if successful, or 1 if not.
7498
7499         Populate the ClassInfo of the core class with copies of the alias and class name, and
7500         with pointers to the relevant portions of the IDL for the core class.
7501 */
7502 static int add_query_core( const char* alias, const char* class_name ) {
7503
7504         // Sanity checks
7505         if( ! curr_query ) {
7506                 osrfLogError( OSRF_LOG_MARK,
7507                                           "%s ERROR: No QueryFrame available for class %s", modulename, class_name );
7508                 return 1;
7509         } else if( curr_query->core.alias ) {
7510                 osrfLogError( OSRF_LOG_MARK,
7511                                           "%s ERROR: Core class %s already populated as %s",
7512                                           modulename, curr_query->core.class_name, curr_query->core.alias );
7513                 return 1;
7514         }
7515
7516         build_class_info( &curr_query->core, alias, class_name );
7517         if( curr_query->core.alias )
7518                 return 0;
7519         else {
7520                 osrfLogError( OSRF_LOG_MARK,
7521                                           "%s ERROR: Unable to look up core class %s", modulename, class_name );
7522                 return 1;
7523         }
7524 }
7525
7526 /**
7527         @brief Search the current QueryFrame for a specified alias.
7528         @param target The alias for which to search.
7529         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7530 */
7531 static inline ClassInfo* search_alias( const char* target ) {
7532         return search_alias_in_frame( curr_query, target );
7533 }
7534
7535 /**
7536         @brief Search all levels of query for a specified alias, starting with the current query.
7537         @param target The alias for which to search.
7538         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7539 */
7540 static ClassInfo* search_all_alias( const char* target ) {
7541         ClassInfo* found_class = NULL;
7542         QueryFrame* curr_frame = curr_query;
7543
7544         while( curr_frame ) {
7545                 if(( found_class = search_alias_in_frame( curr_frame, target ) ))
7546                         break;
7547                 else
7548                         curr_frame = curr_frame->next;
7549         }
7550
7551         return found_class;
7552 }
7553
7554 /**
7555         @brief Add a class to the list of classes joined to the current query.
7556         @param alias Alias of the class to be added.  If it is NULL or an empty string, we use
7557         the class name as an alias.
7558         @param classname The name of the class to be added.
7559         @return A pointer to the ClassInfo for the added class, if successful; otherwise NULL.
7560 */
7561 static ClassInfo* add_joined_class( const char* alias, const char* classname ) {
7562
7563         if( ! classname || ! *classname ) {    // sanity check
7564                 osrfLogError( OSRF_LOG_MARK, "Can't join a class with no class name" );
7565                 return NULL;
7566         }
7567
7568         if( ! alias )
7569                 alias = classname;
7570
7571         const ClassInfo* conflict = search_alias( alias );
7572         if( conflict ) {
7573                 osrfLogError( OSRF_LOG_MARK,
7574                                           "%s ERROR: Table alias \"%s\" conflicts with class \"%s\"",
7575                                           modulename, alias, conflict->class_name );
7576                 return NULL;
7577         }
7578
7579         ClassInfo* info = allocate_class_info();
7580
7581         if( build_class_info( info, alias, classname ) ) {
7582                 free_class_info( info );
7583                 return NULL;
7584         }
7585
7586         // Add the new ClassInfo to the join list of the current QueryFrame
7587         info->next = curr_query->join_list;
7588         curr_query->join_list = info;
7589
7590         return info;
7591 }
7592
7593 /**
7594         @brief Destroy all nodes on the query stack.
7595 */
7596 static void clear_query_stack( void ) {
7597         while( curr_query )
7598                 pop_query_frame();
7599 }
7600
7601 /**
7602         @brief Implement the set_audit_info method.
7603         @param ctx Pointer to the method context.
7604         @return Zero if successful, or -1 if not.
7605
7606         Issue a SAVEPOINT to the database server.
7607
7608         Method parameters:
7609         - authkey
7610         - user id (int)
7611         - workstation id (int)
7612
7613         If user id is not provided the authkey will be used.
7614         For PCRUD the authkey is always used, even if a user is provided.
7615 */
7616 int setAuditInfo( osrfMethodContext* ctx ) {
7617         if(osrfMethodVerifyContext( ctx )) {
7618                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
7619                 return -1;
7620         }
7621
7622         // Get the user id from the parameters
7623         const char* user_id = jsonObjectGetString( jsonObjectGetIndex(ctx->params, 1) );
7624
7625         if( enforce_pcrud || !user_id ) {
7626                 timeout_needs_resetting = 1;
7627                 const jsonObject* user = verifyUserPCRUD( ctx );
7628                 if( !user )
7629                         return -1;
7630                 osrfAppRespondComplete( ctx, NULL );
7631                 return 0;
7632         }
7633
7634         // Not PCRUD and have a user_id?
7635         int result = writeAuditInfo( ctx, user_id, jsonObjectGetString( jsonObjectGetIndex(ctx->params, 2) ) );
7636         osrfAppRespondComplete( ctx, NULL );
7637         return result;
7638 }
7639
7640 /**
7641         @brief Save a audit info
7642         @param ctx Pointer to the method context.
7643         @param user_id User ID to write as a string
7644         @param ws_id Workstation ID to write as a string
7645 */
7646 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id) {
7647         if( ctx && ctx->session ) {
7648                 osrfAppSession* session = ctx->session;
7649
7650                 osrfHash* cache = session->userData;
7651
7652                 // If the session doesn't already have a hash, create one.  Make sure
7653                 // that the application session frees the hash when it terminates.
7654                 if( NULL == cache ) {
7655                         session->userData = cache = osrfNewHash();
7656                         osrfHashSetCallback( cache, &sessionDataFree );
7657                         ctx->session->userDataFree = &userDataFree;
7658                 }
7659
7660                 dbi_result result = dbi_conn_queryf( writehandle, "SELECT auditor.set_audit_info( %s, %s );", user_id, ws_id ? ws_id : "NULL" );
7661                 if( !result ) {
7662                         osrfLogWarning( OSRF_LOG_MARK, "BAD RESULT" );
7663                         const char* msg;
7664                         int errnum = dbi_conn_error( writehandle, &msg );
7665                         osrfLogError(
7666                                 OSRF_LOG_MARK,
7667                                 "%s: Error setting auditor information: %d %s",
7668                                 modulename,
7669                                 errnum,
7670                                 msg ? msg : "(No description available)"
7671                         );
7672                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
7673                                 "osrfMethodException", ctx->request, "Error setting auditor info" );
7674                         if( !oilsIsDBConnected( writehandle ))
7675                                 osrfAppSessionPanic( ctx->session );
7676                         return -1;
7677                 } else {
7678                         dbi_result_free( result );
7679                 }
7680         }
7681         return 0;
7682 }
7683
7684 /**
7685         @brief Remove all but safe character from savepoint name
7686         @param sp User-supplied savepoint name
7687         @return sanitized savepoint name, or NULL
7688
7689     The caller is expected to free the returned string.  Note that
7690     this function exists only because we can't use PQescapeLiteral
7691     without either forking libdbi or abandoning it.
7692 */
7693 static char* _sanitize_savepoint_name( const char* sp ) {
7694
7695         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_";
7696
7697         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7698         // and the default value of NAMEDATALEN is 64; that should be long enough
7699         // for our purposes, and it's unlikely that anyone is going to recompile
7700         // PostgreSQL to have a smaller value, so cap the identifier name
7701         // accordingly to avoid the remote chance that someone manages to pass in a
7702         // 12GB savepoint name
7703         const int MAX_LITERAL_NAMELEN = 63;
7704         int len = 0;
7705         len = strlen( sp );
7706         if (len > MAX_LITERAL_NAMELEN) {
7707                 len = MAX_LITERAL_NAMELEN;
7708         }
7709
7710         char* safeSpName = safe_malloc( len + 1 );
7711         int i = 0;
7712         int j;
7713         char* found;
7714         for (j = 0; j < len; j++) {
7715         found = strchr(safe_chars, sp[j]);
7716                 if (found) {
7717                         safeSpName[ i++ ] = found[0];
7718                 }
7719         }
7720         safeSpName[ i ] = '\0';
7721         return safeSpName;
7722 }
7723
7724 /**
7725         @brief Remove all but safe character from TZ name
7726         @param tz User-supplied TZ name
7727         @return sanitized TZ name, or NULL
7728
7729     The caller is expected to free the returned string.  Note that
7730     this function exists only because we can't use PQescapeLiteral
7731     without either forking libdbi or abandoning it.
7732 */
7733 static char* _sanitize_tz_name( const char* tz ) {
7734
7735         if (NULL == tz) return NULL;
7736
7737         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_/-+";
7738
7739         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7740         // and the default value of NAMEDATALEN is 64; that should be long enough
7741         // for our purposes, and it's unlikely that anyone is going to recompile
7742         // PostgreSQL to have a smaller value, so cap the identifier name
7743         // accordingly to avoid the remote chance that someone manages to pass in a
7744         // 12GB savepoint name
7745         const int MAX_LITERAL_NAMELEN = 63;
7746         int len = 0;
7747         len = strlen( tz );
7748         if (len > MAX_LITERAL_NAMELEN) {
7749                 len = MAX_LITERAL_NAMELEN;
7750         }
7751
7752         char* safeSpName = safe_malloc( len + 1 );
7753         int i = 0;
7754         int j;
7755         char* found;
7756         for (j = 0; j < len; j++) {
7757         found = strchr(safe_chars, tz[j]);
7758                 if (found) {
7759                         safeSpName[ i++ ] = found[0];
7760                 }
7761         }
7762         safeSpName[ i ] = '\0';
7763         return safeSpName;
7764 }
7765
7766 /*@}*/