Open-ILS/src/c-apps/oils_sql.c

   1 /**
   2         @file oils_sql.c
   3         @brief Utility routines for translating JSON into SQL.
   4 */
   5
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <time.h>
   9 #include <ctype.h>
  10 #include <dbi/dbi.h>
  11 #include "opensrf/utils.h"
  12 #include "opensrf/log.h"
  13 #include "opensrf/osrf_application.h"
  14 #include "openils/oils_utils.h"
  15 #include "openils/oils_sql.h"
  16
  17 // The next four macros are OR'd together as needed to form a set
  18 // of bitflags.  SUBCOMBO enables an extra pair of parentheses when
  19 // nesting one UNION, INTERSECT or EXCEPT inside another.
  20 // SUBSELECT tells us we're in a subquery, so don't add the
  21 // terminal semicolon yet.
  22 #define SUBCOMBO    8
  23 #define SUBSELECT   4
  24 #define DISABLE_I18N    2
  25 #define SELECT_DISTINCT 1
  26
  27 #define AND_OP_JOIN     0
  28 #define OR_OP_JOIN      1
  29
  30 struct ClassInfoStruct;
  31 typedef struct ClassInfoStruct ClassInfo;
  32
  33 #define ALIAS_STORE_SIZE 16
  34 #define CLASS_NAME_STORE_SIZE 16
  35
  36 struct ClassInfoStruct {
  37         char* alias;
  38         char* class_name;
  39         char* source_def;
  40         osrfHash* class_def;      // Points into IDL
  41         osrfHash* fields;         // Points into IDL
  42         osrfHash* links;          // Points into IDL
  43
  44         // The remaining members are private and internal.  Client code should not
  45         // access them directly.
  46
  47         ClassInfo* next;          // Supports linked list of joined classes
  48         int in_use;               // boolean
  49
  50         // We usually store the alias and class name in the following arrays, and
  51         // point the corresponding pointers at them.  When the string is too big
  52         // for the array (which will probably never happen in practice), we strdup it.
  53
  54         char alias_store[ ALIAS_STORE_SIZE + 1 ];
  55         char class_name_store[ CLASS_NAME_STORE_SIZE + 1 ];
  56 };
  57
  58 struct QueryFrameStruct;
  59 typedef struct QueryFrameStruct QueryFrame;
  60
  61 struct QueryFrameStruct {
  62         ClassInfo core;
  63         ClassInfo* join_list;  // linked list of classes joined to the core class
  64         QueryFrame* next;      // implements stack as linked list
  65         int in_use;            // boolean
  66 };
  67
  68 static int timeout_needs_resetting;
  69 static time_t time_next_reset;
  70
  71 static int verifyObjectClass ( osrfMethodContext*, const jsonObject* );
  72
  73 static void setXactId( osrfMethodContext* ctx );
  74 static inline const char* getXactId( osrfMethodContext* ctx );
  75 static inline void clearXactId( osrfMethodContext* ctx );
  76
  77 static jsonObject* doFieldmapperSearch ( osrfMethodContext* ctx, osrfHash* class_meta,
  78                 jsonObject* where_hash, jsonObject* query_hash, int* err );
  79 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result, osrfHash* );
  80 static jsonObject* oilsMakeJSONFromResult( dbi_result );
  81
  82 static char* searchSimplePredicate ( const char* op, const char* class_alias,
  83                                 osrfHash* field, const jsonObject* node );
  84 static char* searchFunctionPredicate ( const char*, osrfHash*, const jsonObject*, const char* );
  85 static char* searchFieldTransform ( const char*, osrfHash*, const jsonObject* );
  86 static char* searchFieldTransformPredicate ( const ClassInfo*, osrfHash*, const jsonObject*,
  87                 const char* );
  88 static char* searchBETWEENPredicate ( const char*, osrfHash*, const jsonObject* );
  89 static char* searchINPredicate ( const char*, osrfHash*,
  90                                                                  jsonObject*, const char*, osrfMethodContext* );
  91 static char* searchPredicate ( const ClassInfo*, osrfHash*, jsonObject*, osrfMethodContext* );
  92 static char* searchJOIN ( const jsonObject*, const ClassInfo* left_info );
  93 static char* searchWHERE ( const jsonObject* search_hash, const ClassInfo*, int, osrfMethodContext* );
  94 static char* buildSELECT( const jsonObject*, jsonObject* rest_of_query,
  95         osrfHash* meta, osrfMethodContext* ctx );
  96 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array );
  97
  98 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags );
  99
 100 char* SELECT ( osrfMethodContext*, jsonObject*, const jsonObject*, const jsonObject*,
 101         const jsonObject*, const jsonObject*, const jsonObject*, const jsonObject*, int );
 102
 103 static osrfStringArray* getPermLocationCache( osrfMethodContext*, const char* );
 104 static void setPermLocationCache( osrfMethodContext*, const char*, osrfStringArray* );
 105
 106 void userDataFree( void* );
 107 static void sessionDataFree( char*, void* );
 108 static void pcacheFree( char*, void* );
 109 static int obj_is_true( const jsonObject* obj );
 110 static const char* json_type( int code );
 111 static const char* get_primitive( osrfHash* field );
 112 static const char* get_datatype( osrfHash* field );
 113 static void pop_query_frame( void );
 114 static void push_query_frame( void );
 115 static int add_query_core( const char* alias, const char* class_name );
 116 static inline ClassInfo* search_alias( const char* target );
 117 static ClassInfo* search_all_alias( const char* target );
 118 static ClassInfo* add_joined_class( const char* alias, const char* classname );
 119 static void clear_query_stack( void );
 120
 121 static const jsonObject* verifyUserPCRUD( osrfMethodContext* );
 122 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext*, int );
 123 static int verifyObjectPCRUD( osrfMethodContext*, osrfHash*, const jsonObject*, int );
 124 static const char* org_tree_root( osrfMethodContext* ctx );
 125 static jsonObject* single_hash( const char* key, const char* value );
 126
 127 static int child_initialized = 0;   /* boolean */
 128
 129 static dbi_conn writehandle; /* our MASTER db connection */
 130 static dbi_conn dbhandle; /* our CURRENT db connection */
 131 //static osrfHash * readHandles;
 132
 133 // The following points to the top of a stack of QueryFrames.  It's a little
 134 // confusing because the top level of the query is at the bottom of the stack.
 135 static QueryFrame* curr_query = NULL;
 136
 137 static dbi_conn writehandle; /* our MASTER db connection */
 138 static dbi_conn dbhandle; /* our CURRENT db connection */
 139 //static osrfHash * readHandles;
 140
 141 static int max_flesh_depth = 100;
 142
 143 static int perm_at_threshold = 5;
 144 static int enforce_pcrud = 0;     // Boolean
 145 static char* modulename = NULL;
 146
 147 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id);
 148
 149 static char* _sanitize_tz_name( const char* tz );
 150 static char* _sanitize_savepoint_name( const char* sp );
 151
 152 /**
 153         @brief Connect to the database.
 154         @return A database connection if successful, or NULL if not.
 155 */
 156 dbi_conn oilsConnectDB( const char* mod_name ) {
 157
 158         osrfLogDebug( OSRF_LOG_MARK, "Attempting to initialize libdbi..." );
 159         if( dbi_initialize( NULL ) == -1 ) {
 160                 osrfLogError( OSRF_LOG_MARK, "Unable to initialize libdbi" );
 161                 return NULL;
 162         } else
 163                 osrfLogDebug( OSRF_LOG_MARK, "... libdbi initialized." );
 164
 165         char* driver = osrf_settings_host_value( "/apps/%s/app_settings/driver", mod_name );
 166         char* user   = osrf_settings_host_value( "/apps/%s/app_settings/database/user", mod_name );
 167         char* host   = osrf_settings_host_value( "/apps/%s/app_settings/database/host", mod_name );
 168         char* port   = osrf_settings_host_value( "/apps/%s/app_settings/database/port", mod_name );
 169         char* db     = osrf_settings_host_value( "/apps/%s/app_settings/database/db", mod_name );
 170         char* pw     = osrf_settings_host_value( "/apps/%s/app_settings/database/pw", mod_name );
 171         char* pg_app = osrf_settings_host_value( "/apps/%s/app_settings/database/application_name", mod_name );
 172
 173         osrfLogDebug( OSRF_LOG_MARK, "Attempting to load the database driver [%s]...", driver );
 174         dbi_conn handle = dbi_conn_new( driver );
 175
 176         if( !handle ) {
 177                 osrfLogError( OSRF_LOG_MARK, "Error loading database driver [%s]", driver );
 178                 return NULL;
 179         }
 180         osrfLogDebug( OSRF_LOG_MARK, "Database driver [%s] seems OK", driver );
 181
 182         osrfLogInfo(OSRF_LOG_MARK, "%s connecting to database.  host=%s, "
 183                 "port=%s, user=%s, db=%s", mod_name, host, port, user, db );
 184
 185         if( host )   dbi_conn_set_option( handle, "host", host );
 186         if( port )   dbi_conn_set_option_numeric( handle, "port", atoi( port ));
 187         if( user )   dbi_conn_set_option( handle, "username", user );
 188         if( pw )     dbi_conn_set_option( handle, "password", pw );
 189         if( db )     dbi_conn_set_option( handle, "dbname", db );
 190         if( pg_app ) dbi_conn_set_option( handle, "pgsql_application_name", pg_app );
 191
 192         free( user );
 193         free( host );
 194         free( port );
 195         free( db );
 196         free( pw );
 197         free( pg_app );
 198
 199         if( dbi_conn_connect( handle ) < 0 ) {
 200                 sleep( 1 );
 201                 if( dbi_conn_connect( handle ) < 0 ) {
 202                         const char* msg;
 203                         dbi_conn_error( handle, &msg );
 204                         osrfLogError( OSRF_LOG_MARK, "Error connecting to database: %s",
 205                                 msg ? msg : "(No description available)" );
 206                         return NULL;
 207                 }
 208         }
 209
 210         osrfLogInfo( OSRF_LOG_MARK, "%s successfully connected to the database", mod_name );
 211
 212         return handle;
 213 }
 214
 215 /**
 216         @brief Select some options.
 217         @param module_name: Name of the server.
 218         @param do_pcrud: Boolean.  True if we are to enforce PCRUD permissions.
 219
 220         This source file is used (at this writing) to implement three different servers:
 221         - open-ils.reporter-store
 222         - open-ils.pcrud
 223         - open-ils.cstore
 224
 225         These servers behave mostly the same, but they implement different combinations of
 226         methods, and open-ils.pcrud enforces a permissions scheme that the other two don't.
 227
 228         Here we use the server name in messages to identify which kind of server issued them.
 229         We use do_crud as a boolean to control whether or not to enforce the permissions scheme.
 230 */
 231 void oilsSetSQLOptions( const char* module_name, int do_pcrud, int flesh_depth ) {
 232         if( !module_name )
 233                 module_name = "open-ils.cstore";   // bulletproofing with a default
 234
 235         if( modulename )
 236                 free( modulename );
 237
 238         modulename = strdup( module_name );
 239         enforce_pcrud = do_pcrud;
 240         max_flesh_depth = flesh_depth;
 241 }
 242
 243 /**
 244         @brief Install a database connection.
 245         @param conn Pointer to a database connection.
 246
 247         In some contexts, @a conn may merely provide a driver so that we can process strings
 248         properly, without providing an open database connection.
 249 */
 250 void oilsSetDBConnection( dbi_conn conn ) {
 251         dbhandle = writehandle = conn;
 252 }
 253
 254 /**
 255         @brief Determine whether a database connection is alive.
 256         @param handle Handle for a database connection.
 257         @return 1 if the connection is alive, or zero if it isn't.
 258 */
 259 int oilsIsDBConnected( dbi_conn handle ) {
 260         // Do an innocuous SELECT.  If it succeeds, the database connection is still good.
 261         dbi_result result = dbi_conn_query( handle, "SELECT 1;" );
 262         if( result ) {
 263                 dbi_result_free( result );
 264                 return 1;
 265         } else {
 266                 // This is a terrible, horrible, no good, very bad kludge.
 267                 // Sometimes the SELECT 1 query fails, not because the database connection is dead,
 268                 // but because (due to a previous error) the database is ignoring all commands,
 269                 // even innocuous SELECTs, until the current transaction is rolled back.  The only
 270                 // known way to detect this condition via the dbi library is by looking at the error
 271                 // message.  This approach will break if the language or wording of the message ever
 272                 // changes.
 273                 // Note: the dbi_conn_ping function purports to determine whether the database
 274                 // connection is live, but at this writing this function is unreliable and useless.
 275                 static const char* ok_msg = "ERROR:  current transaction is aborted, commands "
 276                         "ignored until end of transaction block\n";
 277                 const char* msg;
 278                 dbi_conn_error( handle, &msg );
 279                 // Newer versions of dbi_conn_error return codes within the error msg.
 280                 // E.g. 3624914: ERROR:  current transaction is aborted, commands ignored until end of transaction block
 281                 // Substring test should work regardless.
 282                 const char* substr = strstr(msg, ok_msg);
 283                 if( substr == NULL ) {
 284                         osrfLogError( OSRF_LOG_MARK, "Database connection isn't working : %s", msg );
 285                         return 0;
 286                 } else
 287                         return 1;   // ignoring SELECT due to previous error; that's okay
 288         }
 289 }
 290
 291 /**
 292         @brief Get a table name, view name, or subquery for use in a FROM clause.
 293         @param class Pointer to the IDL class entry.
 294         @return A table name, a view name, or a subquery in parentheses.
 295
 296         In some cases the IDL defines a class, not with a table name or a view name, but with
 297         a SELECT statement, which may be used as a subquery.
 298 */
 299 char* oilsGetRelation( osrfHash* classdef ) {
 300
 301         char* source_def = NULL;
 302         const char* tabledef = osrfHashGet( classdef, "tablename" );
 303
 304         if( tabledef ) {
 305                 source_def = strdup( tabledef );   // Return the name of a table or view
 306         } else {
 307                 tabledef = osrfHashGet( classdef, "source_definition" );
 308                 if( tabledef ) {
 309                         // Return a subquery, enclosed in parentheses
 310                         source_def = safe_malloc( strlen( tabledef ) + 3 );
 311                         source_def[ 0 ] = '(';
 312                         strcpy( source_def + 1, tabledef );
 313                         strcat( source_def, ")" );
 314                 } else {
 315                         // Not found: return an error
 316                         const char* classname = osrfHashGet( classdef, "classname" );
 317                         if( !classname )
 318                                 classname = "???";
 319                         osrfLogError(
 320                                 OSRF_LOG_MARK,
 321                                 "%s ERROR No tablename or source_definition for class \"%s\"",
 322                                 modulename,
 323                                 classname
 324                         );
 325                 }
 326         }
 327
 328         return source_def;
 329 }
 330
 331 /**
 332         @brief Add datatypes from the database to the fields in the IDL.
 333         @param handle Handle for a database connection
 334         @return Zero if successful, or 1 upon error.
 335
 336         For each relevant class in the IDL: ask the database for the datatype of every field.
 337         In particular, determine which fields are text fields and which fields are numeric
 338         fields, so that we know whether to enclose their values in quotes.
 339 */
 340 int oilsExtendIDL( dbi_conn handle ) {
 341         osrfHashIterator* class_itr = osrfNewHashIterator( oilsIDL() );
 342         osrfHash* class = NULL;
 343         growing_buffer* query_buf = buffer_init( 64 );
 344         int results_found = 0;   // boolean
 345
 346         // For each class in the IDL...
 347         while( (class = osrfHashIteratorNext( class_itr ) ) ) {
 348                 const char* classname = osrfHashIteratorKey( class_itr );
 349                 osrfHash* fields = osrfHashGet( class, "fields" );
 350
 351                 // If the class is virtual, ignore it
 352                 if( str_is_true( osrfHashGet(class, "virtual") ) ) {
 353                         osrfLogDebug(OSRF_LOG_MARK, "Class %s is virtual, skipping", classname );
 354                         continue;
 355                 }
 356
 357                 char* tabledef = oilsGetRelation( class );
 358                 if( !tabledef )
 359                         continue;   // No such relation -- a query of it would be doomed to failure
 360
 361                 buffer_reset( query_buf );
 362                 buffer_fadd( query_buf, "SELECT * FROM %s AS x WHERE 1=0;", tabledef );
 363
 364                 free(tabledef );
 365
 366                 osrfLogDebug( OSRF_LOG_MARK, "%s Investigatory SQL = %s",
 367                                 modulename, OSRF_BUFFER_C_STR( query_buf ) );
 368
 369                 dbi_result result = dbi_conn_query( handle, OSRF_BUFFER_C_STR( query_buf ) );
 370                 if( result ) {
 371
 372                         results_found = 1;
 373                         int columnIndex = 1;
 374                         const char* columnName;
 375                         while( (columnName = dbi_result_get_field_name(result, columnIndex)) ) {
 376
 377                                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...",
 378                                                 columnName );
 379
 380                                 /* fetch the fieldmapper index */
 381                                 osrfHash* _f = osrfHashGet(fields, columnName);
 382                                 if( _f ) {
 383
 384                                         osrfLogDebug(OSRF_LOG_MARK, "Found [%s] in IDL hash...", columnName);
 385
 386                                         /* determine the field type and storage attributes */
 387
 388                                         switch( dbi_result_get_field_type_idx( result, columnIndex )) {
 389
 390                                                 case DBI_TYPE_INTEGER : {
 391
 392                                                         if( !osrfHashGet(_f, "primitive") )
 393                                                                 osrfHashSet(_f, "number", "primitive");
 394
 395                                                         int attr = dbi_result_get_field_attribs_idx( result, columnIndex );
 396                                                         if( attr & DBI_INTEGER_SIZE8 )
 397                                                                 osrfHashSet( _f, "INT8", "datatype" );
 398                                                         else
 399                                                                 osrfHashSet( _f, "INT", "datatype" );
 400                                                         break;
 401                                                 }
 402                                                 case DBI_TYPE_DECIMAL :
 403                                                         if( !osrfHashGet( _f, "primitive" ))
 404                                                                 osrfHashSet( _f, "number", "primitive" );
 405
 406                                                         osrfHashSet( _f, "NUMERIC", "datatype" );
 407                                                         break;
 408
 409                                                 case DBI_TYPE_STRING :
 410                                                         if( !osrfHashGet( _f, "primitive" ))
 411                                                                 osrfHashSet( _f, "string", "primitive" );
 412
 413                                                         osrfHashSet( _f,"TEXT", "datatype" );
 414                                                         break;
 415
 416                                                 case DBI_TYPE_DATETIME :
 417                                                         if( !osrfHashGet( _f, "primitive" ))
 418                                                                 osrfHashSet( _f, "string", "primitive" );
 419
 420                                                         osrfHashSet( _f, "TIMESTAMP", "datatype" );
 421                                                         break;
 422
 423                                                 case DBI_TYPE_BINARY :
 424                                                         if( !osrfHashGet( _f, "primitive" ))
 425                                                                 osrfHashSet( _f, "string", "primitive" );
 426
 427                                                         osrfHashSet( _f, "BYTEA", "datatype" );
 428                                         }
 429
 430                                         osrfLogDebug(
 431                                                 OSRF_LOG_MARK,
 432                                                 "Setting [%s] to primitive [%s] and datatype [%s]...",
 433                                                 columnName,
 434                                                 osrfHashGet( _f, "primitive" ),
 435                                                 osrfHashGet( _f, "datatype" )
 436                                         );
 437                                 }
 438                                 ++columnIndex;
 439                         } // end while loop for traversing columns of result
 440                         dbi_result_free( result  );
 441                 } else {
 442                         const char* msg;
 443                         int errnum = dbi_conn_error( handle, &msg );
 444                         osrfLogDebug( OSRF_LOG_MARK, "No data found for class [%s]: %d, %s", classname,
 445                                 errnum, msg ? msg : "(No description available)" );
 446                         // We don't check the database connection here.  It's routine to get failures at
 447                         // this point; we routinely try to query tables that don't exist, because they
 448                         // are defined in the IDL but not in the database.
 449                 }
 450         } // end for each class in IDL
 451
 452         buffer_free( query_buf );
 453         osrfHashIteratorFree( class_itr );
 454         child_initialized = 1;
 455
 456         if( !results_found ) {
 457                 osrfLogError( OSRF_LOG_MARK,
 458                         "No results found for any class -- bad database connection?" );
 459                 return 1;
 460         } else if( ! oilsIsDBConnected( handle )) {
 461                 osrfLogError( OSRF_LOG_MARK,
 462                         "Unable to extend IDL: database connection isn't working" );
 463                 return 1;
 464         }
 465         else
 466                 return 0;
 467 }
 468
 469 /**
 470         @brief Free an osrfHash that stores a transaction ID.
 471         @param blob A pointer to the osrfHash to be freed, cast to a void pointer.
 472
 473         This function is a callback, to be called by the application session when it ends.
 474         The application session stores the osrfHash via an opaque pointer.
 475
 476         If the osrfHash contains an entry for the key "xact_id", it means that an
 477         uncommitted transaction is pending.  Roll it back.
 478 */
 479 void userDataFree( void* blob ) {
 480         osrfHash* hash = (osrfHash*) blob;
 481         if( osrfHashGet( hash, "xact_id" ) && writehandle ) {
 482                 if( !dbi_conn_query( writehandle, "ROLLBACK;" )) {
 483                         const char* msg;
 484                         int errnum = dbi_conn_error( writehandle, &msg );
 485                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform rollback: %d %s",
 486                                 errnum, msg ? msg : "(No description available)" );
 487                 };
 488         }
 489         if( writehandle ) {
 490                 if( !dbi_conn_query( writehandle, "SELECT auditor.clear_audit_info();" ) ) {
 491                         const char* msg;
 492                         int errnum = dbi_conn_error( writehandle, &msg );
 493                         osrfLogWarning( OSRF_LOG_MARK, "Unable to perform audit info clearing: %d %s",
 494                                 errnum, msg ? msg : "(No description available)" );
 495                 }
 496         }
 497
 498         osrfHashFree( hash );
 499 }
 500
 501 /**
 502         @name Managing session data
 503         @brief Maintain data stored via the userData pointer of the application session.
 504
 505         Currently, session-level data is stored in an osrfHash.  Other arrangements are
 506         possible, and some would be more efficient.  The application session calls a
 507         callback function to free userData before terminating.
 508
 509         Currently, the only data we store at the session level is the transaction id.  By this
 510         means we can ensure that any pending transactions are rolled back before the application
 511         session terminates.
 512 */
 513 /*@{*/
 514
 515 /**
 516         @brief Free an item in the application session's userData.
 517         @param key The name of a key for an osrfHash.
 518         @param item An opaque pointer to the item associated with the key.
 519
 520         We store an osrfHash as userData with the application session, and arrange (by
 521         installing userDataFree() as a different callback) for the session to free that
 522         osrfHash before terminating.
 523
 524         This function is a callback for freeing items in the osrfHash.  Currently we store
 525         two things:
 526         - Transaction id of a pending transaction; a character string.  Key: "xact_id".
 527         - Authkey; a character string.  Key: "authkey".
 528         - User object from the authentication server; a jsonObject.  Key: "user_login".
 529
 530         If we ever store anything else in userData, we will need to revisit this function so
 531         that it will free whatever else needs freeing.
 532 */
 533 static void sessionDataFree( char* key, void* item ) {
 534         if( !strcmp( key, "xact_id" ) || !strcmp( key, "authkey" ) || !strncmp( key, "rs_size_", 8) )
 535                 free( item );
 536         else if( !strcmp( key, "user_login" ) )
 537                 jsonObjectFree( (jsonObject*) item );
 538         else if( !strcmp( key, "pcache" ) )
 539                 osrfHashFree( (osrfHash*) item );
 540 }
 541
 542 static void pcacheFree( char* key, void* item ) {
 543         osrfStringArrayFree( (osrfStringArray*) item );
 544 }
 545
 546 /**
 547         @brief Initialize session cache.
 548         @param ctx Pointer to the method context.
 549
 550         Create a cache for the session by making the session's userData member point
 551         to an osrfHash instance.
 552 */
 553 static osrfHash* initSessionCache( osrfMethodContext* ctx ) {
 554         ctx->session->userData = osrfNewHash();
 555         osrfHashSetCallback( (osrfHash*) ctx->session->userData, &sessionDataFree );
 556         ctx->session->userDataFree = &userDataFree;
 557         return ctx->session->userData;
 558 }
 559
 560 /**
 561         @brief Save a transaction id.
 562         @param ctx Pointer to the method context.
 563
 564         Save the session_id of the current application session as a transaction id.
 565 */
 566 static void setXactId( osrfMethodContext* ctx ) {
 567         if( ctx && ctx->session ) {
 568                 osrfAppSession* session = ctx->session;
 569
 570                 osrfHash* cache = session->userData;
 571
 572                 // If the session doesn't already have a hash, create one.  Make sure
 573                 // that the application session frees the hash when it terminates.
 574                 if( NULL == cache )
 575                         cache = initSessionCache( ctx );
 576
 577                 // Save the transaction id in the hash, with the key "xact_id"
 578                 osrfHashSet( cache, strdup( session->session_id ), "xact_id" );
 579         }
 580 }
 581
 582 /**
 583         @brief Get the transaction ID for the current transaction, if any.
 584         @param ctx Pointer to the method context.
 585         @return Pointer to the transaction ID.
 586
 587         The return value points to an internal buffer, and will become invalid upon issuing
 588         a commit or rollback.
 589 */
 590 static inline const char* getXactId( osrfMethodContext* ctx ) {
 591         if( ctx && ctx->session && ctx->session->userData )
 592                 return osrfHashGet( (osrfHash*) ctx->session->userData, "xact_id" );
 593         else
 594                 return NULL;
 595 }
 596
 597 /**
 598         @brief Clear the current transaction id.
 599         @param ctx Pointer to the method context.
 600 */
 601 static inline void clearXactId( osrfMethodContext* ctx ) {
 602         if( ctx && ctx->session && ctx->session->userData )
 603                 osrfHashRemove( ctx->session->userData, "xact_id" );
 604 }
 605 /*@}*/
 606
 607 /**
 608         @brief Stash the location for a particular perm in the sessionData cache
 609         @param ctx Pointer to the method context.
 610         @param perm Name of the permission we're looking at
 611         @param array StringArray of perm location ids
 612 */
 613 static void setPermLocationCache( osrfMethodContext* ctx, const char* perm, osrfStringArray* locations ) {
 614         if( ctx && ctx->session ) {
 615                 osrfAppSession* session = ctx->session;
 616
 617                 osrfHash* cache = session->userData;
 618
 619                 // If the session doesn't already have a hash, create one.  Make sure
 620                 // that the application session frees the hash when it terminates.
 621                 if( NULL == cache )
 622                         cache = initSessionCache( ctx );
 623
 624                 osrfHash* pcache = osrfHashGet(cache, "pcache");
 625
 626                 if( NULL == pcache ) {
 627                         pcache = osrfNewHash();
 628                         osrfHashSetCallback( pcache, &pcacheFree );
 629                         osrfHashSet( cache, pcache, "pcache" );
 630                 }
 631
 632                 if( perm && locations )
 633                         osrfHashSet( pcache, locations, strdup(perm) );
 634         }
 635 }
 636
 637 /**
 638         @brief Grab stashed location for a particular perm in the sessionData cache
 639         @param ctx Pointer to the method context.
 640         @param perm Name of the permission we're looking at
 641 */
 642 static osrfStringArray* getPermLocationCache( osrfMethodContext* ctx, const char* perm ) {
 643         if( ctx && ctx->session ) {
 644                 osrfAppSession* session = ctx->session;
 645                 osrfHash* cache = session->userData;
 646                 if( cache ) {
 647                         osrfHash* pcache = osrfHashGet(cache, "pcache");
 648                         if( pcache ) {
 649                                 return osrfHashGet( pcache, perm );
 650                         }
 651                 }
 652         }
 653
 654         return NULL;
 655 }
 656
 657 /**
 658         @brief Save the user's login in the userData for the current application session.
 659         @param ctx Pointer to the method context.
 660         @param user_login Pointer to the user login object to be cached (we cache the original,
 661         not a copy of it).
 662
 663         If @a user_login is NULL, remove the user login if one is already cached.
 664 */
 665 static void setUserLogin( osrfMethodContext* ctx, jsonObject* user_login ) {
 666         if( ctx && ctx->session ) {
 667                 osrfAppSession* session = ctx->session;
 668
 669                 osrfHash* cache = session->userData;
 670
 671                 // If the session doesn't already have a hash, create one.  Make sure
 672                 // that the application session frees the hash when it terminates.
 673                 if( NULL == cache )
 674                         cache = initSessionCache( ctx );
 675
 676                 if( user_login )
 677                         osrfHashSet( cache, user_login, "user_login" );
 678                 else
 679                         osrfHashRemove( cache, "user_login" );
 680         }
 681 }
 682
 683 /**
 684         @brief Get the user login object for the current application session, if any.
 685         @param ctx Pointer to the method context.
 686         @return Pointer to the user login object if found; otherwise NULL.
 687
 688         The user login object was returned from the authentication server, and then cached so
 689         we don't have to call the authentication server again for the same user.
 690 */
 691 static const jsonObject* getUserLogin( osrfMethodContext* ctx ) {
 692         if( ctx && ctx->session && ctx->session->userData )
 693                 return osrfHashGet( (osrfHash*) ctx->session->userData, "user_login" );
 694         else
 695                 return NULL;
 696 }
 697
 698 /**
 699         @brief Save a copy of an authkey in the userData of the current application session.
 700         @param ctx Pointer to the method context.
 701         @param authkey The authkey to be saved.
 702
 703         If @a authkey is NULL, remove the authkey if one is already cached.
 704 */
 705 static void setAuthkey( osrfMethodContext* ctx, const char* authkey ) {
 706         if( ctx && ctx->session && authkey ) {
 707                 osrfAppSession* session = ctx->session;
 708                 osrfHash* cache = session->userData;
 709
 710                 // If the session doesn't already have a hash, create one.  Make sure
 711                 // that the application session frees the hash when it terminates.
 712                 if( NULL == cache )
 713                         cache = initSessionCache( ctx );
 714
 715                 // Save the transaction id in the hash, with the key "xact_id"
 716                 if( authkey && *authkey )
 717                         osrfHashSet( cache, strdup( authkey ), "authkey" );
 718                 else
 719                         osrfHashRemove( cache, "authkey" );
 720         }
 721 }
 722
 723 /**
 724         @brief Reset the login timeout.
 725         @param authkey The authentication key for the current login session.
 726         @param now The current time.
 727         @return Zero if successful, or 1 if not.
 728
 729         Tell the authentication server to reset the timeout so that the login session won't
 730         expire for a while longer.
 731
 732         We could dispense with the @a now parameter by calling time().  But we just called
 733         time() in order to decide whether to reset the timeout, so we might as well reuse
 734         the result instead of calling time() again.
 735 */
 736 static int reset_timeout( const char* authkey, time_t now ) {
 737         jsonObject* auth_object = jsonNewObject( authkey );
 738
 739         // Ask the authentication server to reset the timeout.  It returns an event
 740         // indicating success or failure.
 741         jsonObject* result = oilsUtilsQuickReq( "open-ils.auth",
 742                 "open-ils.auth.session.reset_timeout", auth_object );
 743         jsonObjectFree( auth_object );
 744
 745         if( !result || result->type != JSON_HASH ) {
 746                 osrfLogError( OSRF_LOG_MARK,
 747                          "Unexpected object type receieved from open-ils.auth.session.reset_timeout" );
 748                 jsonObjectFree( result );
 749                 return 1;       // Not the right sort of object returned
 750         }
 751
 752         const jsonObject* ilsevent = jsonObjectGetKeyConst( result, "ilsevent" );
 753         if( !ilsevent || ilsevent->type != JSON_NUMBER ) {
 754                 osrfLogError( OSRF_LOG_MARK, "ilsevent is absent or malformed" );
 755                 jsonObjectFree( result );
 756                 return 1;    // Return code from method not available
 757         }
 758
 759         if( jsonObjectGetNumber( ilsevent ) != 0.0 ) {
 760                 const char* desc = jsonObjectGetString( jsonObjectGetKeyConst( result, "desc" ));
 761                 if( !desc )
 762                         desc = "(No reason available)";    // failsafe; shouldn't happen
 763                 osrfLogInfo( OSRF_LOG_MARK, "Failure to reset timeout: %s", desc );
 764                 jsonObjectFree( result );
 765                 return 1;
 766         }
 767
 768         // Revise our local proxy for the timeout deadline
 769         // by a smallish fraction of the timeout interval
 770         const char* timeout = jsonObjectGetString( jsonObjectGetKeyConst( result, "payload" ));
 771         if( !timeout )
 772                 timeout = "1";   // failsafe; shouldn't happen
 773         time_next_reset = now + atoi( timeout ) / 15;
 774
 775         jsonObjectFree( result );
 776         return 0;     // Successfully reset timeout
 777 }
 778
 779 /**
 780         @brief Get the authkey string for the current application session, if any.
 781         @param ctx Pointer to the method context.
 782         @return Pointer to the cached authkey if found; otherwise NULL.
 783
 784         If present, the authkey string was cached from a previous method call.
 785 */
 786 static const char* getAuthkey( osrfMethodContext* ctx ) {
 787         if( ctx && ctx->session && ctx->session->userData ) {
 788                 const char* authkey = osrfHashGet( (osrfHash*) ctx->session->userData, "authkey" );
 789         // LFW recent changes mean the userData hash gets set up earlier, but
 790         // doesn't necessarily have an authkey yet
 791         if (!authkey)
 792             return NULL;
 793
 794                 // Possibly reset the authentication timeout to keep the login alive.  We do so
 795                 // no more than once per method call, and not at all if it has been only a short
 796                 // time since the last reset.
 797
 798                 // Here we reset explicitly, if at all.  We also implicitly reset the timeout
 799                 // whenever we call the "open-ils.auth.session.retrieve" method.
 800                 if( timeout_needs_resetting ) {
 801                         time_t now = time( NULL );
 802                         if( now >= time_next_reset && reset_timeout( authkey, now ) )
 803                                 authkey = NULL;    // timeout has apparently expired already
 804                 }
 805
 806                 timeout_needs_resetting = 0;
 807                 return authkey;
 808         }
 809         else
 810                 return NULL;
 811 }
 812
 813 /**
 814         @brief Implement the transaction.begin method.
 815         @param ctx Pointer to the method context.
 816         @return Zero if successful, or -1 upon error.
 817
 818         Start a transaction.  Save a transaction ID for future reference.
 819
 820         Method parameters:
 821         - authkey (PCRUD only)
 822
 823         Return to client: Transaction ID
 824 */
 825 int beginTransaction( osrfMethodContext* ctx ) {
 826         if(osrfMethodVerifyContext( ctx )) {
 827                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 828                 return -1;
 829         }
 830
 831         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
 832
 833         if( enforce_pcrud ) {
 834                 timeout_needs_resetting = 1;
 835                 const jsonObject* user = verifyUserPCRUD( ctx );
 836                 if( !user )
 837                         return -1;
 838         }
 839
 840         dbi_result result = dbi_conn_query( writehandle, "START TRANSACTION;" );
 841         if( !result ) {
 842                 const char* msg;
 843                 int errnum = dbi_conn_error( writehandle, &msg );
 844                 osrfLogError( OSRF_LOG_MARK, "%s: Error starting transaction: %d %s",
 845                         modulename, errnum, msg ? msg : "(No description available)" );
 846                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 847                         "osrfMethodException", ctx->request, "Error starting transaction" );
 848                 if( !oilsIsDBConnected( writehandle ))
 849                         osrfAppSessionPanic( ctx->session );
 850                 return -1;
 851         } else {
 852                 dbi_result_free( result );
 853                 setXactId( ctx );
 854                 jsonObject* ret = jsonNewObject( getXactId( ctx ) );
 855                 osrfAppRespondComplete( ctx, ret );
 856                 jsonObjectFree( ret );
 857
 858         }
 859
 860         if (tz) {
 861                 setenv("TZ",tz,1);
 862                 dbi_result tz_res = dbi_conn_queryf( writehandle, "SET LOCAL timezone TO '%s'; -- cstore", tz );
 863                 if( !tz_res ) {
 864                         osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
 865                         if( !oilsIsDBConnected( writehandle )) {
 866                                 osrfAppSessionPanic( ctx->session );
 867                                 return -1;
 868                         }
 869                 } else {
 870                         dbi_result_free( tz_res );
 871                 }
 872         } else {
 873                 unsetenv("TZ");
 874                 dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- no tz" );
 875                 if( !res ) {
 876                         osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
 877                         if( !oilsIsDBConnected( writehandle )) {
 878                                 osrfAppSessionPanic( ctx->session );
 879                                 return -1;
 880                         }
 881                 } else {
 882                         dbi_result_free( res );
 883                 }
 884         }
 885
 886         return 0;
 887 }
 888
 889 /**
 890         @brief Implement the savepoint.set method.
 891         @param ctx Pointer to the method context.
 892         @return Zero if successful, or -1 if not.
 893
 894         Issue a SAVEPOINT to the database server.
 895
 896         Method parameters:
 897         - authkey (PCRUD only)
 898         - savepoint name
 899
 900         Return to client: Savepoint name
 901 */
 902 int setSavepoint( osrfMethodContext* ctx ) {
 903         if(osrfMethodVerifyContext( ctx )) {
 904                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 905                 return -1;
 906         }
 907
 908         int spNamePos = 0;
 909         if( enforce_pcrud ) {
 910                 spNamePos = 1;
 911                 timeout_needs_resetting = 1;
 912                 const jsonObject* user = verifyUserPCRUD( ctx );
 913                 if( !user )
 914                         return -1;
 915         }
 916
 917         // Verify that a transaction is pending
 918         const char* trans_id = getXactId( ctx );
 919         if( NULL == trans_id ) {
 920                 osrfAppSessionStatus(
 921                         ctx->session,
 922                         OSRF_STATUS_INTERNALSERVERERROR,
 923                         "osrfMethodException",
 924                         ctx->request,
 925                         "No active transaction -- required for savepoints"
 926                 );
 927                 return -1;
 928         }
 929
 930         // Get the savepoint name from the method params
 931         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
 932
 933         if (!spName) {
 934                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.set called with no name");
 935                 return -1;
 936         }
 937
 938         char *safeSpName = _sanitize_savepoint_name( spName );
 939
 940         dbi_result result = dbi_conn_queryf( writehandle, "SAVEPOINT \"%s\";", safeSpName );
 941         free( safeSpName );
 942         if( !result ) {
 943                 const char* msg;
 944                 int errnum = dbi_conn_error( writehandle, &msg );
 945                 osrfLogError(
 946                         OSRF_LOG_MARK,
 947                         "%s: Error creating savepoint %s in transaction %s: %d %s",
 948                         modulename,
 949                         spName,
 950                         trans_id,
 951                         errnum,
 952                         msg ? msg : "(No description available)"
 953                 );
 954                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
 955                         "osrfMethodException", ctx->request, "Error creating savepoint" );
 956                 if( !oilsIsDBConnected( writehandle ))
 957                         osrfAppSessionPanic( ctx->session );
 958                 return -1;
 959         } else {
 960                 dbi_result_free( result );
 961                 jsonObject* ret = jsonNewObject( spName );
 962                 osrfAppRespondComplete( ctx, ret );
 963                 jsonObjectFree( ret  );
 964                 return 0;
 965         }
 966 }
 967
 968 /**
 969         @brief Implement the savepoint.release method.
 970         @param ctx Pointer to the method context.
 971         @return Zero if successful, or -1 if not.
 972
 973         Issue a RELEASE SAVEPOINT to the database server.
 974
 975         Method parameters:
 976         - authkey (PCRUD only)
 977         - savepoint name
 978
 979         Return to client: Savepoint name
 980 */
 981 int releaseSavepoint( osrfMethodContext* ctx ) {
 982         if(osrfMethodVerifyContext( ctx )) {
 983                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
 984                 return -1;
 985         }
 986
 987         int spNamePos = 0;
 988         if( enforce_pcrud ) {
 989                 spNamePos = 1;
 990                 timeout_needs_resetting = 1;
 991                 const jsonObject* user = verifyUserPCRUD( ctx );
 992                 if(  !user )
 993                         return -1;
 994         }
 995
 996         // Verify that a transaction is pending
 997         const char* trans_id = getXactId( ctx );
 998         if( NULL == trans_id ) {
 999                 osrfAppSessionStatus(
1000                         ctx->session,
1001                         OSRF_STATUS_INTERNALSERVERERROR,
1002                         "osrfMethodException",
1003                         ctx->request,
1004                         "No active transaction -- required for savepoints"
1005                 );
1006                 return -1;
1007         }
1008
1009         // Get the savepoint name from the method params
1010         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1011
1012         if (!spName) {
1013                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.release called with no name");
1014                 return -1;
1015         }
1016
1017         char *safeSpName = _sanitize_savepoint_name( spName );
1018
1019         dbi_result result = dbi_conn_queryf( writehandle, "RELEASE SAVEPOINT \"%s\";", safeSpName );
1020         free( safeSpName );
1021         if( !result ) {
1022                 const char* msg;
1023                 int errnum = dbi_conn_error( writehandle, &msg );
1024                 osrfLogError(
1025                         OSRF_LOG_MARK,
1026                         "%s: Error releasing savepoint %s in transaction %s: %d %s",
1027                         modulename,
1028                         spName,
1029                         trans_id,
1030                         errnum,
1031                         msg ? msg : "(No description available)"
1032                 );
1033                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1034                         "osrfMethodException", ctx->request, "Error releasing savepoint" );
1035                 if( !oilsIsDBConnected( writehandle ))
1036                         osrfAppSessionPanic( ctx->session );
1037                 return -1;
1038         } else {
1039                 dbi_result_free( result );
1040                 jsonObject* ret = jsonNewObject( spName );
1041                 osrfAppRespondComplete( ctx, ret );
1042                 jsonObjectFree( ret );
1043                 return 0;
1044         }
1045 }
1046
1047 /**
1048         @brief Implement the savepoint.rollback method.
1049         @param ctx Pointer to the method context.
1050         @return Zero if successful, or -1 if not.
1051
1052         Issue a ROLLBACK TO SAVEPOINT to the database server.
1053
1054         Method parameters:
1055         - authkey (PCRUD only)
1056         - savepoint name
1057
1058         Return to client: Savepoint name
1059 */
1060 int rollbackSavepoint( osrfMethodContext* ctx ) {
1061         if(osrfMethodVerifyContext( ctx )) {
1062                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1063                 return -1;
1064         }
1065
1066         int spNamePos = 0;
1067         if( enforce_pcrud ) {
1068                 spNamePos = 1;
1069                 timeout_needs_resetting = 1;
1070                 const jsonObject* user = verifyUserPCRUD( ctx );
1071                 if( !user )
1072                         return -1;
1073         }
1074
1075         // Verify that a transaction is pending
1076         const char* trans_id = getXactId( ctx );
1077         if( NULL == trans_id ) {
1078                 osrfAppSessionStatus(
1079                         ctx->session,
1080                         OSRF_STATUS_INTERNALSERVERERROR,
1081                         "osrfMethodException",
1082                         ctx->request,
1083                         "No active transaction -- required for savepoints"
1084                 );
1085                 return -1;
1086         }
1087
1088         // Get the savepoint name from the method params
1089         const char* spName = jsonObjectGetString( jsonObjectGetIndex(ctx->params, spNamePos) );
1090
1091         if (!spName) {
1092                 osrfLogWarning(OSRF_LOG_MARK, "savepoint.rollback called with no name");
1093                 return -1;
1094         }
1095
1096         char *safeSpName = _sanitize_savepoint_name( spName );
1097
1098         dbi_result result = dbi_conn_queryf( writehandle, "ROLLBACK TO SAVEPOINT \"%s\";", safeSpName );
1099         free( safeSpName );
1100         if( !result ) {
1101                 const char* msg;
1102                 int errnum = dbi_conn_error( writehandle, &msg );
1103                 osrfLogError(
1104                         OSRF_LOG_MARK,
1105                         "%s: Error rolling back savepoint %s in transaction %s: %d %s",
1106                         modulename,
1107                         spName,
1108                         trans_id,
1109                         errnum,
1110                         msg ? msg : "(No description available)"
1111                 );
1112                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1113                         "osrfMethodException", ctx->request, "Error rolling back savepoint" );
1114                 if( !oilsIsDBConnected( writehandle ))
1115                         osrfAppSessionPanic( ctx->session );
1116                 return -1;
1117         } else {
1118                 dbi_result_free( result );
1119                 jsonObject* ret = jsonNewObject( spName );
1120                 osrfAppRespondComplete( ctx, ret );
1121                 jsonObjectFree( ret );
1122                 return 0;
1123         }
1124 }
1125
1126 /**
1127         @brief Implement the transaction.commit method.
1128         @param ctx Pointer to the method context.
1129         @return Zero if successful, or -1 if not.
1130
1131         Issue a COMMIT to the database server.
1132
1133         Method parameters:
1134         - authkey (PCRUD only)
1135
1136         Return to client: Transaction ID.
1137 */
1138 int commitTransaction( osrfMethodContext* ctx ) {
1139         if(osrfMethodVerifyContext( ctx )) {
1140                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1141                 return -1;
1142         }
1143
1144         if( enforce_pcrud ) {
1145                 timeout_needs_resetting = 1;
1146                 const jsonObject* user = verifyUserPCRUD( ctx );
1147                 if( !user )
1148                         return -1;
1149         }
1150
1151         // Verify that a transaction is pending
1152         const char* trans_id = getXactId( ctx );
1153         if( NULL == trans_id ) {
1154                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1155                                 "osrfMethodException", ctx->request, "No active transaction to commit" );
1156                 return -1;
1157         }
1158
1159         dbi_result result = dbi_conn_query( writehandle, "COMMIT;" );
1160         if( !result ) {
1161                 const char* msg;
1162                 int errnum = dbi_conn_error( writehandle, &msg );
1163                 osrfLogError( OSRF_LOG_MARK, "%s: Error committing transaction: %d %s",
1164                         modulename, errnum, msg ? msg : "(No description available)" );
1165                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1166                         "osrfMethodException", ctx->request, "Error committing transaction" );
1167                 if( !oilsIsDBConnected( writehandle ))
1168                         osrfAppSessionPanic( ctx->session );
1169                 return -1;
1170         } else {
1171                 dbi_result_free( result );
1172                 jsonObject* ret = jsonNewObject( trans_id );
1173                 osrfAppRespondComplete( ctx, ret );
1174                 jsonObjectFree( ret );
1175                 clearXactId( ctx );
1176                 return 0;
1177         }
1178 }
1179
1180 /**
1181         @brief Implement the transaction.rollback method.
1182         @param ctx Pointer to the method context.
1183         @return Zero if successful, or -1 if not.
1184
1185         Issue a ROLLBACK to the database server.
1186
1187         Method parameters:
1188         - authkey (PCRUD only)
1189
1190         Return to client: Transaction ID
1191 */
1192 int rollbackTransaction( osrfMethodContext* ctx ) {
1193         if( osrfMethodVerifyContext( ctx )) {
1194                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
1195                 return -1;
1196         }
1197
1198         if( enforce_pcrud ) {
1199                 timeout_needs_resetting = 1;
1200                 const jsonObject* user = verifyUserPCRUD( ctx );
1201                 if( !user )
1202                         return -1;
1203         }
1204
1205         // Verify that a transaction is pending
1206         const char* trans_id = getXactId( ctx );
1207         if( NULL == trans_id ) {
1208                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1209                                 "osrfMethodException", ctx->request, "No active transaction to roll back" );
1210                 return -1;
1211         }
1212
1213         dbi_result result = dbi_conn_query( writehandle, "ROLLBACK;" );
1214         if( !result ) {
1215                 const char* msg;
1216                 int errnum = dbi_conn_error( writehandle, &msg );
1217                 osrfLogError( OSRF_LOG_MARK, "%s: Error rolling back transaction: %d %s",
1218                         modulename, errnum, msg ? msg : "(No description available)" );
1219                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
1220                         "osrfMethodException", ctx->request, "Error rolling back transaction" );
1221                 if( !oilsIsDBConnected( writehandle ))
1222                         osrfAppSessionPanic( ctx->session );
1223                 return -1;
1224         } else {
1225                 dbi_result_free( result );
1226                 jsonObject* ret = jsonNewObject( trans_id );
1227                 osrfAppRespondComplete( ctx, ret );
1228                 jsonObjectFree( ret );
1229                 clearXactId( ctx );
1230                 return 0;
1231         }
1232 }
1233
1234 /**
1235         @brief Implement the "search" method.
1236         @param ctx Pointer to the method context.
1237         @return Zero if successful, or -1 if not.
1238
1239         Method parameters:
1240         - authkey (PCRUD only)
1241         - WHERE clause, as jsonObject
1242         - Other SQL clause(s), as a JSON_HASH: joins, SELECT list, LIMIT, etc.
1243
1244         Return to client: rows of the specified class that satisfy a specified WHERE clause.
1245         Optionally flesh linked fields.
1246 */
1247 int doSearch( osrfMethodContext* ctx ) {
1248         if( osrfMethodVerifyContext( ctx )) {
1249                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1250                 return -1;
1251         }
1252
1253         if( enforce_pcrud )
1254                 timeout_needs_resetting = 1;
1255
1256         jsonObject* where_clause;
1257         jsonObject* rest_of_query;
1258
1259         if( enforce_pcrud ) {
1260                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1261                 rest_of_query = jsonObjectGetIndex( ctx->params, 2 );
1262         } else {
1263                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1264                 rest_of_query = jsonObjectGetIndex( ctx->params, 1 );
1265         }
1266
1267         if( !where_clause ) {
1268                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1269                 return -1;
1270         }
1271
1272         // Get the class metadata
1273         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1274         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1275
1276         // Do the query
1277         int err = 0;
1278         jsonObject* obj = doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1279         if( err ) {
1280                 osrfAppRespondComplete( ctx, NULL );
1281                 return -1;
1282         }
1283
1284         // doFieldmapperSearch() now takes care of our responding for us
1285 //      // Return each row to the client
1286 //      jsonObject* cur = 0;
1287 //      unsigned long res_idx = 0;
1288 //
1289 //      while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1290 //              // We used to discard based on perms here, but now that's
1291 //              // inside doFieldmapperSearch()
1292 //              osrfAppRespond( ctx, cur );
1293 //      }
1294
1295         jsonObjectFree( obj );
1296
1297         osrfAppRespondComplete( ctx, NULL );
1298         return 0;
1299 }
1300
1301 /**
1302         @brief Implement the "id_list" method.
1303         @param ctx Pointer to the method context.
1304         @param err Pointer through which to return an error code.
1305         @return Zero if successful, or -1 if not.
1306
1307         Method parameters:
1308         - authkey (PCRUD only)
1309         - WHERE clause, as jsonObject
1310         - Other SQL clause(s), as a JSON_HASH: joins, LIMIT, etc.
1311
1312         Return to client: The primary key values for all rows of the relevant class that
1313         satisfy a specified WHERE clause.
1314
1315         This method relies on the assumption that every class has a primary key consisting of
1316         a single column.
1317 */
1318 int doIdList( osrfMethodContext* ctx ) {
1319         if( osrfMethodVerifyContext( ctx )) {
1320                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
1321                 return -1;
1322         }
1323
1324         if( enforce_pcrud )
1325                 timeout_needs_resetting = 1;
1326
1327         jsonObject* where_clause;
1328         jsonObject* rest_of_query;
1329
1330         // We use the where clause without change.  But we need to massage the rest of the
1331         // query, so we work with a copy of it instead of modifying the original.
1332
1333         if( enforce_pcrud ) {
1334                 where_clause  = jsonObjectGetIndex( ctx->params, 1 );
1335                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 2 ) );
1336         } else {
1337                 where_clause  = jsonObjectGetIndex( ctx->params, 0 );
1338                 rest_of_query = jsonObjectClone( jsonObjectGetIndex( ctx->params, 1 ) );
1339         }
1340
1341         if( !where_clause ) {
1342                 osrfLogError( OSRF_LOG_MARK, "No WHERE clause parameter supplied" );
1343                 return -1;
1344         }
1345
1346         // Eliminate certain SQL clauses, if present.
1347         if( rest_of_query ) {
1348                 jsonObjectRemoveKey( rest_of_query, "select" );
1349                 jsonObjectRemoveKey( rest_of_query, "no_i18n" );
1350                 jsonObjectRemoveKey( rest_of_query, "flesh" );
1351                 jsonObjectRemoveKey( rest_of_query, "flesh_fields" );
1352         } else {
1353                 rest_of_query = jsonNewObjectType( JSON_HASH );
1354         }
1355
1356         jsonObjectSetKey( rest_of_query, "no_i18n", jsonNewBoolObject( 1 ) );
1357
1358         // Get the class metadata
1359         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1360         osrfHash* class_meta = osrfHashGet( method_meta, "class" );
1361
1362         // Build a SELECT list containing just the primary key,
1363         // i.e. like { "classname":["keyname"] }
1364         jsonObject* col_list_obj = jsonNewObjectType( JSON_ARRAY );
1365
1366         // Load array with name of primary key
1367         jsonObjectPush( col_list_obj, jsonNewObject( osrfHashGet( class_meta, "primarykey" ) ) );
1368         jsonObject* select_clause = jsonNewObjectType( JSON_HASH );
1369         jsonObjectSetKey( select_clause, osrfHashGet( class_meta, "classname" ), col_list_obj );
1370
1371         jsonObjectSetKey( rest_of_query, "select", select_clause );
1372
1373         // Do the query
1374         int err = 0;
1375         jsonObject* obj =
1376                 doFieldmapperSearch( ctx, class_meta, where_clause, rest_of_query, &err );
1377
1378         jsonObjectFree( rest_of_query );
1379         if( err ) {
1380                 osrfAppRespondComplete( ctx, NULL );
1381                 return -1;
1382         }
1383
1384         // Return each primary key value to the client
1385         jsonObject* cur;
1386         unsigned long res_idx = 0;
1387         while((cur = jsonObjectGetIndex( obj, res_idx++ ) )) {
1388                 // We used to discard based on perms here, but now that's
1389                 // inside doFieldmapperSearch()
1390                 osrfAppRespond( ctx,
1391                         oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
1392         }
1393
1394         jsonObjectFree( obj );
1395         osrfAppRespondComplete( ctx, NULL );
1396         return 0;
1397 }
1398
1399 /**
1400         @brief Verify that we have a valid class reference.
1401         @param ctx Pointer to the method context.
1402         @param param Pointer to the method parameters.
1403         @return 1 if the class reference is valid, or zero if it isn't.
1404
1405         The class of the method params must match the class to which the method id devoted.
1406         For PCRUD there are additional restrictions.
1407 */
1408 static int verifyObjectClass ( osrfMethodContext* ctx, const jsonObject* param ) {
1409
1410         osrfHash* method_meta = (osrfHash*) ctx->method->userData;
1411         osrfHash* class = osrfHashGet( method_meta, "class" );
1412
1413         // Compare the method's class to the parameters' class
1414         if( !param->classname || (strcmp( osrfHashGet(class, "classname"), param->classname ))) {
1415
1416                 // Oops -- they don't match.  Complain.
1417                 growing_buffer* msg = buffer_init( 128 );
1418                 buffer_fadd(
1419                         msg,
1420                         "%s: %s method for type %s was passed a %s",
1421                         modulename,
1422                         osrfHashGet( method_meta, "methodtype" ),
1423                         osrfHashGet( class, "classname" ),
1424                         param->classname ? param->classname : "(null)"
1425                 );
1426
1427                 char* m = buffer_release( msg );
1428                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_BADREQUEST, "osrfMethodException",
1429                                 ctx->request, m );
1430                 free( m );
1431
1432                 return 0;
1433         }
1434
1435         if( enforce_pcrud )
1436                 return verifyObjectPCRUD( ctx, class, param, 1 );
1437         else
1438                 return 1;
1439 }
1440
1441 /**
1442         @brief (PCRUD only) Verify that the user is properly logged in.
1443         @param ctx Pointer to the method context.
1444         @return If the user is logged in, a pointer to the user object from the authentication
1445         server; otherwise NULL.
1446 */
1447 static const jsonObject* verifyUserPCRUD( osrfMethodContext* ctx ) {
1448         return verifyUserPCRUDfull( ctx, 0 );
1449 }
1450
1451 static const jsonObject* verifyUserPCRUDfull( osrfMethodContext* ctx, int anon_ok ) {
1452
1453         // Get the authkey (the first method parameter)
1454         const char* auth = jsonObjectGetString( jsonObjectGetIndex( ctx->params, 0 ) );
1455
1456         jsonObject* user = NULL;
1457
1458         // If we are /not/ in anonymous mode
1459         if( strcmp( "ANONYMOUS", auth ) ) {
1460                 // See if we have the same authkey, and a user object,
1461                 // locally cached from a previous call
1462                 const char* cached_authkey = getAuthkey( ctx );
1463                 if( cached_authkey && !strcmp( cached_authkey, auth ) ) {
1464                         const jsonObject* cached_user = getUserLogin( ctx );
1465                         if( cached_user )
1466                                 return cached_user;
1467                 }
1468
1469                 // We have no matching authentication data in the cache.  Authenticate from scratch.
1470                 jsonObject* auth_object = jsonNewObject( auth );
1471
1472                 // Fetch the user object from the authentication server
1473                 user = oilsUtilsQuickReq( "open-ils.auth", "open-ils.auth.session.retrieve", auth_object );
1474                 jsonObjectFree( auth_object );
1475
1476                 if( !user->classname || strcmp(user->classname, "au" )) {
1477
1478                         growing_buffer* msg = buffer_init( 128 );
1479                         buffer_fadd(
1480                                 msg,
1481                                 "%s: permacrud received a bad auth token: %s",
1482                                 modulename,
1483                                 auth
1484                         );
1485
1486                         char* m = buffer_release( msg );
1487                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_UNAUTHORIZED, "osrfMethodException",
1488                                         ctx->request, m );
1489
1490                         free( m );
1491                         jsonObjectFree( user );
1492                         user = NULL;
1493                 } else if( writeAuditInfo( ctx, oilsFMGetStringConst( user, "id" ), oilsFMGetStringConst( user, "wsid" ) ) ) {
1494                         // Failed to set audit information - But note that write_audit_info already set error information.
1495                         jsonObjectFree( user );
1496                         user = NULL;
1497                 }
1498
1499
1500         } else if ( anon_ok ) { // we /are/ (attempting to be) anonymous
1501                 user = jsonNewObjectType(JSON_ARRAY);
1502                 jsonObjectSetClass( user, "aou" );
1503                 oilsFMSetString(user, "id", "-1");
1504         }
1505
1506         setUserLogin( ctx, user );
1507         setAuthkey( ctx, auth );
1508
1509         // Allow ourselves up to a second before we have to reset the login timeout.
1510         // It would be nice to use some fraction of the timeout interval enforced by the
1511         // authentication server, but that value is not readily available at this point.
1512         // Instead, we use a conservative default interval.
1513         time_next_reset = time( NULL ) + 1;
1514
1515         return user;
1516 }
1517
1518 /**
1519         @brief For PCRUD: Determine whether the current user may access the current row.
1520         @param ctx Pointer to the method context.
1521         @param class Same as ctx->method->userData's item for key "class" except when called in recursive doFieldmapperSearch
1522         @param obj Pointer to the row being potentially accessed.
1523         @return 1 if access is permitted, or 0 if it isn't.
1524
1525         The @a obj parameter points to a JSON_HASH of column values, keyed on column name.
1526 */
1527 static int verifyObjectPCRUD ( osrfMethodContext* ctx, osrfHash *class, const jsonObject* obj, int rs_size ) {
1528
1529         dbhandle = writehandle;
1530
1531         // Figure out what class and method are involved
1532         osrfHash* method_metadata = (osrfHash*) ctx->method->userData;
1533         const char* method_type = osrfHashGet( method_metadata, "methodtype" );
1534
1535         if (!rs_size) {
1536                 int *rs_size_from_hash = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
1537                 if (rs_size_from_hash) {
1538                         rs_size = *rs_size_from_hash;
1539                         osrfLogDebug(OSRF_LOG_MARK, "used rs_size from request-scoped hash: %d", rs_size);
1540                 }
1541         }
1542
1543         // Set fetch to 1 in all cases except for inserts, meaning that for local or foreign
1544         // contexts we will do another lookup of the current row, even if we already have a
1545         // previously fetched row image, because the row image in hand may not include the
1546         // foreign key(s) that we need.
1547
1548         // This is a quick fix with a bludgeon.  There are ways to avoid the extra lookup,
1549         // but they aren't implemented yet.
1550
1551         int fetch = 0;
1552         if( *method_type == 's' || *method_type == 'i' ) {
1553                 method_type = "retrieve"; // search and id_list are equivalent to retrieve for this
1554                 fetch = 1;
1555         } else if( *method_type == 'u' || *method_type == 'd' ) {
1556                 fetch = 1; // MUST go to the db for the object for update and delete
1557         }
1558
1559         // In retrieve or search ONLY we allow anon. Later perm checks will fail as they should,
1560         // in the face of a fake user but required permissions.
1561         int anon_ok = 0;
1562         if( *method_type == 'r' )
1563                 anon_ok = 1;
1564
1565         // Get the appropriate permacrud entry from the IDL, depending on method type
1566         osrfHash* pcrud = osrfHashGet( osrfHashGet( class, "permacrud" ), method_type );
1567         if( !pcrud ) {
1568                 // No permacrud for this method type on this class
1569
1570                 growing_buffer* msg = buffer_init( 128 );
1571                 buffer_fadd(
1572                         msg,
1573                         "%s: %s on class %s has no permacrud IDL entry",
1574                         modulename,
1575                         osrfHashGet( method_metadata, "methodtype" ),
1576                         osrfHashGet( class, "classname" )
1577                 );
1578
1579                 char* m = buffer_release( msg );
1580                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_FORBIDDEN,
1581                                 "osrfMethodException", ctx->request, m );
1582
1583                 free( m );
1584
1585                 return 0;
1586         }
1587
1588         // Get the user id, and make sure the user is logged in
1589         const jsonObject* user = verifyUserPCRUDfull( ctx, anon_ok );
1590         if( !user )
1591                 return 0;    // Not logged in or anon?  No access.
1592
1593         int userid = atoi( oilsFMGetStringConst( user, "id" ) );
1594
1595         // Get a list of permissions from the permacrud entry.
1596         osrfStringArray* permission = osrfHashGet( pcrud, "permission" );
1597         if( permission->size == 0 ) {
1598                 osrfLogDebug(
1599                         OSRF_LOG_MARK,
1600                         "No permissions required for this action (class %s), passing through",
1601                         osrfHashGet(class, "classname")
1602                 );
1603                 return 1;
1604         }
1605
1606         // But, if there are perms and the user is anonymous ... FAIL
1607         if ( -1 == userid )
1608                 return 0;
1609
1610         // Build a list of org units that own the row.  This is fairly convoluted because there
1611         // are several different ways that an org unit may own the row, as defined by the
1612         // permacrud entry.
1613
1614         // Local context means that the row includes a foreign key pointing to actor.org_unit,
1615         // identifying an owning org_unit..
1616         osrfStringArray* local_context = osrfHashGet( pcrud, "local_context" );
1617
1618         // Foreign context adds a layer of indirection.  The row points to some other row that
1619         // an org unit may own.  The "jump" attribute, if present, adds another layer of
1620         // indirection.
1621         osrfHash* foreign_context = osrfHashGet( pcrud, "foreign_context" );
1622
1623         // The following string array stores the list of org units.  (We don't have a thingie
1624         // for storing lists of integers, so we fake it with a list of strings.)
1625         osrfStringArray* context_org_array = osrfNewStringArray( 1 );
1626
1627         const char* context_org = NULL;
1628     const char* pkey = NULL;
1629     jsonObject *param = NULL;
1630         const char* perm = NULL;
1631         int OK = 0;
1632         int i = 0;
1633         int err = 0;
1634         const char* pkey_value = NULL;
1635         if( str_is_true( osrfHashGet(pcrud, "global_required") ) ) {
1636                 // If the global_required attribute is present and true, then the only owning
1637                 // org unit is the root org unit, i.e. the one with no parent.
1638                 osrfLogDebug( OSRF_LOG_MARK,
1639                                 "global-level permissions required, fetching top of the org tree" );
1640
1641                 // no need to check perms for org tree root retrieval
1642                 osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1643                 // check for perm at top of org tree
1644                 const char* org_tree_root_id = org_tree_root( ctx );
1645                 osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1646
1647                 if( org_tree_root_id ) {
1648                         osrfStringArrayAdd( context_org_array, org_tree_root_id );
1649                         osrfLogDebug( OSRF_LOG_MARK, "top of the org tree is %s", org_tree_root_id );
1650                 } else  {
1651                         osrfStringArrayFree( context_org_array );
1652                         return 0;
1653                 }
1654
1655         } else {
1656                 // If the global_required attribute is absent or false, then we look for
1657                 // local and/or foreign context.  In order to find the relevant foreign
1658                 // keys, we must either read the relevant row from the database, or look at
1659                 // the image of the row that we already have in memory.
1660
1661                 // Even if we have an image of the row in memory, that image may not include the
1662                 // foreign key column(s) that we need.  So whenever possible, we do a fresh read
1663                 // of the row to make sure that we have what we need.
1664
1665             osrfLogDebug( OSRF_LOG_MARK, "global-level permissions not required, "
1666                                 "fetching context org ids" );
1667
1668         pkey = osrfHashGet( class, "primarykey" );
1669
1670                 if( !pkey ) {
1671                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1672                         // image that we already have.  If it doesn't have everything we need, too bad.
1673                         fetch = 0;
1674                         param = jsonObjectClone( obj );
1675                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1676                 } else if( obj->classname ) {
1677                         pkey_value = oilsFMGetStringConst( obj, pkey );
1678                         if( !fetch )
1679                                 param = jsonObjectClone( obj );
1680                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1681                                 pkey_value );
1682                 } else {
1683                         pkey_value = jsonObjectGetString( obj );
1684                         fetch = 1;
1685                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1686                                 "of %s and retrieving from the database", pkey_value );
1687                 }
1688
1689                 if( fetch ) {
1690                         // Fetch the row so that we can look at the foreign key(s)
1691                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1692                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1693                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1694                         jsonObjectFree( _tmp_params );
1695                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1696
1697                         param = jsonObjectExtractIndex( _list, 0 );
1698                         jsonObjectFree( _list );
1699
1700             fetch = 0;
1701                 }
1702
1703                 if( !param ) {
1704                         // The row doesn't exist.  Complain, and deny access.
1705                         osrfLogDebug( OSRF_LOG_MARK,
1706                                         "Object not found in the database with primary key %s of %s",
1707                                         pkey, pkey_value );
1708
1709                         growing_buffer* msg = buffer_init( 128 );
1710                         buffer_fadd(
1711                                 msg,
1712                                 "%s: no object found with primary key %s of %s",
1713                                 modulename,
1714                                 pkey,
1715                                 pkey_value
1716                         );
1717
1718                         char* m = buffer_release( msg );
1719                         osrfAppSessionStatus(
1720                                 ctx->session,
1721                                 OSRF_STATUS_INTERNALSERVERERROR,
1722                                 "osrfMethodException",
1723                                 ctx->request,
1724                                 m
1725                         );
1726
1727                         free( m );
1728                         return 0;
1729                 }
1730
1731                 if( local_context && local_context->size > 0 ) {
1732                         // The IDL provides a list of column names for the foreign keys denoting
1733                         // local context, i.e. columns identifying owing org units directly.  Look up
1734                         // the value of each one, and if it isn't null, add it to the list of org units.
1735                         osrfLogDebug( OSRF_LOG_MARK, "%d class-local context field(s) specified",
1736                                 local_context->size );
1737                         int i = 0;
1738                         const char* lcontext = NULL;
1739                         while ( (lcontext = osrfStringArrayGetString(local_context, i++)) ) {
1740                                 const char* fkey_value = oilsFMGetStringConst( param, lcontext );
1741                                 if( fkey_value ) {    // if not null
1742                                         osrfStringArrayAdd( context_org_array, fkey_value );
1743                                         osrfLogDebug(
1744                                                 OSRF_LOG_MARK,
1745                                                 "adding class-local field %s (value: %s) to the context org list",
1746                                                 lcontext,
1747                                                 osrfStringArrayGetString( context_org_array, context_org_array->size - 1 )
1748                                         );
1749                                 }
1750                         }
1751                 }
1752
1753                 if( foreign_context ) {
1754                         unsigned long class_count = osrfHashGetCount( foreign_context );
1755                         osrfLogDebug( OSRF_LOG_MARK, "%d foreign context classes(s) specified", class_count );
1756
1757                         if( class_count > 0 ) {
1758
1759                                 // The IDL provides a list of foreign key columns pointing to rows that
1760                                 // an org unit may own.  Follow each link, identify the owning org unit,
1761                                 // and add it to the list.
1762                                 osrfHash* fcontext = NULL;
1763                                 osrfHashIterator* class_itr = osrfNewHashIterator( foreign_context );
1764                                 while( (fcontext = osrfHashIteratorNext( class_itr )) ) {
1765                                         // For each class to which a foreign key points:
1766                                         const char* class_name = osrfHashIteratorKey( class_itr );
1767                                         osrfHash* fcontext = osrfHashGet( foreign_context, class_name );
1768
1769                                         osrfLogDebug(
1770                                                 OSRF_LOG_MARK,
1771                                                 "%d foreign context fields(s) specified for class %s",
1772                                                 ((osrfStringArray*)osrfHashGet(fcontext,"context"))->size,
1773                                                 class_name
1774                                         );
1775
1776                                         // Get the name of the key field in the foreign table
1777                                         const char* foreign_pkey = osrfHashGet( fcontext, "field" );
1778
1779                                         // Get the value of the foreign key pointing to the foreign table
1780                                         char* foreign_pkey_value =
1781                                                         oilsFMGetString( param, osrfHashGet( fcontext, "fkey" ));
1782                                         if( !foreign_pkey_value )
1783                                                 continue;    // Foreign key value is null; skip it
1784
1785                                         // Look up the row to which the foreign key points
1786                                         jsonObject* _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1787
1788                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1789                                         jsonObject* _list = doFieldmapperSearch(
1790                                                 ctx, osrfHashGet( oilsIDL(), class_name ), _tmp_params, NULL, &err );
1791                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1792
1793                                         jsonObject* _fparam = NULL;
1794                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1795                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1796
1797                                         jsonObjectFree( _tmp_params );
1798                                         jsonObjectFree( _list );
1799
1800                                         // At this point _fparam either points to the row identified by the
1801                                         // foreign key, or it's NULL (no such row found).
1802
1803                                         osrfStringArray* jump_list = osrfHashGet( fcontext, "jump" );
1804
1805                                         const char* bad_class = NULL;  // For noting failed lookups
1806                                         if( ! _fparam )
1807                                                 bad_class = class_name;    // Referenced row not found
1808                                         else if( jump_list ) {
1809                                                 // Follow a chain of rows, linked by foreign keys, to find an owner
1810                                                 const char* flink = NULL;
1811                                                 int k = 0;
1812                                                 while ( (flink = osrfStringArrayGetString(jump_list, k++)) && _fparam ) {
1813                                                         // For each entry in the jump list.  Each entry (i.e. flink) is
1814                                                         // the name of a foreign key column in the current row.
1815
1816                                                         // From the IDL, get the linkage information for the next jump
1817                                                         osrfHash* foreign_link_hash =
1818                                                                         oilsIDLFindPath( "/%s/links/%s", _fparam->classname, flink );
1819
1820                                                         // Get the class metadata for the class
1821                                                         // to which the foreign key points
1822                                                         osrfHash* foreign_class_meta = osrfHashGet( oilsIDL(),
1823                                                                         osrfHashGet( foreign_link_hash, "class" ));
1824
1825                                                         // Get the name of the referenced key of that class
1826                                                         foreign_pkey = osrfHashGet( foreign_link_hash, "key" );
1827
1828                                                         // Get the value of the foreign key pointing to that class
1829                                                         free( foreign_pkey_value );
1830                                                         foreign_pkey_value = oilsFMGetString( _fparam, flink );
1831                                                         if( !foreign_pkey_value )
1832                                                                 break;    // Foreign key is null; quit looking
1833
1834                                                         // Build a WHERE clause for the lookup
1835                                                         _tmp_params = single_hash( foreign_pkey, foreign_pkey_value );
1836
1837                                                         // Do the lookup
1838                                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1839                                                         _list = doFieldmapperSearch( ctx, foreign_class_meta,
1840                                                                         _tmp_params, NULL, &err );
1841                                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1842
1843                                                         // Get the resulting row
1844                                                         jsonObjectFree( _fparam );
1845                                                         if( _list && JSON_ARRAY == _list->type && _list->size > 0 )
1846                                                                 _fparam = jsonObjectExtractIndex( _list, 0 );
1847                                                         else {
1848                                                                 // Referenced row not found
1849                                                                 _fparam = NULL;
1850                                                                 bad_class = osrfHashGet( foreign_link_hash, "class" );
1851                                                         }
1852
1853                                                         jsonObjectFree( _tmp_params );
1854                                                         jsonObjectFree( _list );
1855                                                 }
1856                                         }
1857
1858                                         if( bad_class ) {
1859
1860                                                 // We had a foreign key pointing to such-and-such a row, but then
1861                                                 // we couldn't fetch that row.  The data in the database are in an
1862                                                 // inconsistent state; the database itself may even be corrupted.
1863                                                 growing_buffer* msg = buffer_init( 128 );
1864                                                 buffer_fadd(
1865                                                         msg,
1866                                                         "%s: no object of class %s found with primary key %s of %s",
1867                                                         modulename,
1868                                                         bad_class,
1869                                                         foreign_pkey,
1870                                                         foreign_pkey_value ? foreign_pkey_value : "(null)"
1871                                                 );
1872
1873                                                 char* m = buffer_release( msg );
1874                                                 osrfAppSessionStatus(
1875                                                         ctx->session,
1876                                                         OSRF_STATUS_INTERNALSERVERERROR,
1877                                                         "osrfMethodException",
1878                                                         ctx->request,
1879                                                         m
1880                                                 );
1881
1882                                                 free( m );
1883                                                 osrfHashIteratorFree( class_itr );
1884                                                 free( foreign_pkey_value );
1885                                                 jsonObjectFree( param );
1886
1887                                                 return 0;
1888                                         }
1889
1890                                         free( foreign_pkey_value );
1891
1892                                         if( _fparam ) {
1893                                                 // Examine each context column of the foreign row,
1894                                                 // and add its value to the list of org units.
1895                                                 int j = 0;
1896                                                 const char* foreign_field = NULL;
1897                                                 osrfStringArray* ctx_array = osrfHashGet( fcontext, "context" );
1898                                                 while ( (foreign_field = osrfStringArrayGetString( ctx_array, j++ )) ) {
1899                                                         osrfStringArrayAdd( context_org_array,
1900                                                                 oilsFMGetStringConst( _fparam, foreign_field ));
1901                                                         osrfLogDebug( OSRF_LOG_MARK,
1902                                                                 "adding foreign class %s field %s (value: %s) "
1903                                                                         "to the context org list",
1904                                                                 class_name,
1905                                                                 foreign_field,
1906                                                                 osrfStringArrayGetString(
1907                                                                         context_org_array, context_org_array->size - 1 )
1908                                                         );
1909                                                 }
1910
1911                                                 jsonObjectFree( _fparam );
1912                                         }
1913                                 }
1914
1915                                 osrfHashIteratorFree( class_itr );
1916                         }
1917                 }
1918         }
1919
1920     // If there is an owning_user attached to the action, we allow that user and users with
1921     // object perms on the object. CREATE can't use this. We only do this when we're not
1922     // ignoring object perms.
1923     char* owning_user_field = osrfHashGet( pcrud, "owning_user" );
1924     if (
1925         *method_type != 'c' &&
1926         (!str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) || // Always honor
1927         owning_user_field)
1928     ) {
1929         if (owning_user_field) { // see if we can short-cut by comparing the owner to the requestor
1930
1931             if (!param) { // We didn't get it during the context lookup
1932                             pkey = osrfHashGet( class, "primarykey" );
1933
1934                                 if( !pkey  ) {
1935                                         // There is no primary key, so we can't do a fresh lookup.  Use the row
1936                                         // image that we already have.  If it doesn't have everything we need, too bad.
1937                                         fetch = 0;
1938                                         param = jsonObjectClone( obj );
1939                                         osrfLogDebug( OSRF_LOG_MARK, "No primary key; using clone of object" );
1940                                 } else if( obj->classname ) {
1941                                         pkey_value = oilsFMGetStringConst( obj, pkey );
1942                                         if( !fetch )
1943                                                 param = jsonObjectClone( obj );
1944                                         osrfLogDebug( OSRF_LOG_MARK, "Object supplied, using primary key value of %s",
1945                                                 pkey_value );
1946                                 } else {
1947                                         pkey_value = jsonObjectGetString( obj );
1948                                         fetch = 1;
1949                                         osrfLogDebug( OSRF_LOG_MARK, "Object not supplied, using primary key value "
1950                                                 "of %s and retrieving from the database", pkey_value );
1951                                 }
1952
1953                                 if( fetch ) {
1954                                         // Fetch the row so that we can look at the foreign key(s)
1955                                         osrfHashSet((osrfHash*) ctx->session->userData, "1", "inside_verify");
1956                                         jsonObject* _tmp_params = single_hash( pkey, pkey_value );
1957                                         jsonObject* _list = doFieldmapperSearch( ctx, class, _tmp_params, NULL, &err );
1958                                         jsonObjectFree( _tmp_params );
1959                                         osrfHashSet((osrfHash*) ctx->session->userData, "0", "inside_verify");
1960
1961                                         param = jsonObjectExtractIndex( _list, 0 );
1962                                         jsonObjectFree( _list );
1963                                 }
1964             }
1965
1966                         if( !param ) {
1967                                 // The row doesn't exist.  Complain, and deny access.
1968                                 osrfLogDebug( OSRF_LOG_MARK,
1969                                                 "Object not found in the database with primary key %s of %s",
1970                                                 pkey, pkey_value );
1971
1972                                 growing_buffer* msg = buffer_init( 128 );
1973                                 buffer_fadd(
1974                                         msg,
1975                                         "%s: no object found with primary key %s of %s",
1976                                         modulename,
1977                                         pkey,
1978                                         pkey_value
1979                                 );
1980
1981                                 char* m = buffer_release( msg );
1982                                 osrfAppSessionStatus(
1983                                         ctx->session,
1984                                         OSRF_STATUS_INTERNALSERVERERROR,
1985                                         "osrfMethodException",
1986                                         ctx->request,
1987                                         m
1988                                 );
1989
1990                                 free( m );
1991                                 return 0;
1992                         } else {
1993
1994                 int ownerid = atoi( oilsFMGetStringConst( param, owning_user_field ) );
1995
1996                 // Allow the owner to do whatever
1997                 if (ownerid == userid)
1998                     OK = 1;
1999                         }
2000                 }
2001
2002         i = 0;
2003         while(  !OK &&
2004                                 (perm = osrfStringArrayGetString(permission, i++)) &&
2005                                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms"))
2006                 ) {
2007                 dbi_result result;
2008
2009                         osrfLogDebug(
2010                                 OSRF_LOG_MARK,
2011                                 "Checking object permission [%s] for user %d "
2012                                                 "on object %s (class %s)",
2013                                 perm,
2014                                 userid,
2015                                 pkey_value,
2016                                 osrfHashGet( class, "classname" )
2017                         );
2018
2019                         result = dbi_conn_queryf(
2020                                 writehandle,
2021                                 "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s') AS has_perm;",
2022                                 userid,
2023                                 perm,
2024                                 osrfHashGet( class, "classname" ),
2025                                 pkey_value
2026                         );
2027
2028                         if( result ) {
2029                                 osrfLogDebug(
2030                                         OSRF_LOG_MARK,
2031                                         "Received a result for object permission [%s] "
2032                                                         "for user %d on object %s (class %s)",
2033                                         perm,
2034                                         userid,
2035                                         pkey_value,
2036                                         osrfHashGet( class, "classname" )
2037                                 );
2038
2039                                 if( dbi_result_first_row( result )) {
2040                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2041                                         const char* has_perm = jsonObjectGetString(
2042                                                         jsonObjectGetKeyConst( return_val, "has_perm" ));
2043
2044                                         osrfLogDebug(
2045                                                 OSRF_LOG_MARK,
2046                                                 "Status of object permission [%s] for user %d "
2047                                                                 "on object %s (class %s) is %s",
2048                                                 perm,
2049                                                 userid,
2050                                                 pkey_value,
2051                                                 osrfHashGet(class, "classname"),
2052                                                 has_perm
2053                                         );
2054
2055                                         if( *has_perm == 't' )
2056                                                 OK = 1;
2057                                         jsonObjectFree( return_val );
2058                                 }
2059
2060                                 dbi_result_free( result );
2061                                 if( OK )
2062                     break;
2063                         } else {
2064                                 const char* msg;
2065                                 int errnum = dbi_conn_error( writehandle, &msg );
2066                                 osrfLogWarning( OSRF_LOG_MARK,
2067                                         "Unable to call check object permissions: %d, %s",
2068                                         errnum, msg ? msg : "(No description available)" );
2069                                 if( !oilsIsDBConnected( writehandle ))
2070                                         osrfAppSessionPanic( ctx->session );
2071                         }
2072         }
2073     }
2074
2075         // For every combination of permission and context org unit: call a stored procedure
2076         // to determine if the user has this permission in the context of this org unit.
2077         // If the answer is yes at any point, then we're done, and the user has permission.
2078         // In other words permissions are additive.
2079         i = 0;
2080         while( !OK && (perm = osrfStringArrayGetString(permission, i++)) ) {
2081                 dbi_result result;
2082
2083         osrfStringArray* pcache = NULL;
2084         if (rs_size > perm_at_threshold) { // grab and cache locations of user perms
2085                         pcache = getPermLocationCache(ctx, perm);
2086
2087                         if (!pcache) {
2088                         pcache = osrfNewStringArray(0);
2089
2090                                 result = dbi_conn_queryf(
2091                                         writehandle,
2092                                         "SELECT permission.usr_has_perm_at_all(%d, '%s') AS at;",
2093                                         userid,
2094                                         perm
2095                                 );
2096
2097                                 if( result ) {
2098                                         osrfLogDebug(
2099                                                 OSRF_LOG_MARK,
2100                                                 "Received a result for permission [%s] for user %d",
2101                                                 perm,
2102                                                 userid
2103                                         );
2104
2105                                         if( dbi_result_first_row( result )) {
2106                             do {
2107                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2108                                                 osrfStringArrayAdd( pcache, jsonObjectGetString( jsonObjectGetKeyConst( return_val, "at" ) ) );
2109                                 jsonObjectFree( return_val );
2110                                             } while( dbi_result_next_row( result ));
2111
2112                                                 setPermLocationCache(ctx, perm, pcache);
2113                                         }
2114
2115                                         dbi_result_free( result );
2116                     }
2117                         }
2118         }
2119
2120                 int j = 0;
2121                 while( (context_org = osrfStringArrayGetString( context_org_array, j++ )) ) {
2122
2123             if (rs_size > perm_at_threshold) {
2124                 if (osrfStringArrayContains( pcache, context_org )) {
2125                     OK = 1;
2126                     break;
2127                 }
2128             }
2129
2130                         if(
2131                 pkey_value &&
2132                 !str_is_true( osrfHashGet(pcrud, "ignore_object_perms") ) && // Always honor
2133                 (
2134                     !str_is_true( osrfHashGet(pcrud, "global_required") ) ||
2135                     osrfHashGet(pcrud, "owning_user")
2136                 )
2137             ) {
2138                                 osrfLogDebug(
2139                                         OSRF_LOG_MARK,
2140                                         "Checking object permission [%s] for user %d "
2141                                                         "on object %s (class %s) at org %d",
2142                                         perm,
2143                                         userid,
2144                                         pkey_value,
2145                                         osrfHashGet( class, "classname" ),
2146                                         atoi( context_org )
2147                                 );
2148
2149                                 result = dbi_conn_queryf(
2150                                         writehandle,
2151                                         "SELECT permission.usr_has_object_perm(%d, '%s', '%s', '%s', %d) AS has_perm;",
2152                                         userid,
2153                                         perm,
2154                                         osrfHashGet( class, "classname" ),
2155                                         pkey_value,
2156                                         atoi( context_org )
2157                                 );
2158
2159                                 if( result ) {
2160                                         osrfLogDebug(
2161                                                 OSRF_LOG_MARK,
2162                                                 "Received a result for object permission [%s] "
2163                                                                 "for user %d on object %s (class %s) at org %d",
2164                                                 perm,
2165                                                 userid,
2166                                                 pkey_value,
2167                                                 osrfHashGet( class, "classname" ),
2168                                                 atoi( context_org )
2169                                         );
2170
2171                                         if( dbi_result_first_row( result )) {
2172                                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
2173                                                 const char* has_perm = jsonObjectGetString(
2174                                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2175
2176                                                 osrfLogDebug(
2177                                                         OSRF_LOG_MARK,
2178                                                         "Status of object permission [%s] for user %d "
2179                                                                         "on object %s (class %s) at org %d is %s",
2180                                                         perm,
2181                                                         userid,
2182                                                         pkey_value,
2183                                                         osrfHashGet(class, "classname"),
2184                                                         atoi(context_org),
2185                                                         has_perm
2186                                                 );
2187
2188                                                 if( *has_perm == 't' )
2189                                                         OK = 1;
2190                                                 jsonObjectFree( return_val );
2191                                         }
2192
2193                                         dbi_result_free( result );
2194                                         if( OK )
2195                                                 break;
2196                                 } else {
2197                                         const char* msg;
2198                                         int errnum = dbi_conn_error( writehandle, &msg );
2199                                         osrfLogWarning( OSRF_LOG_MARK,
2200                                                 "Unable to call check object permissions: %d, %s",
2201                                                 errnum, msg ? msg : "(No description available)" );
2202                                         if( !oilsIsDBConnected( writehandle ))
2203                                                 osrfAppSessionPanic( ctx->session );
2204                                 }
2205                         }
2206
2207             if (rs_size > perm_at_threshold) break;
2208
2209                         osrfLogDebug( OSRF_LOG_MARK,
2210                                         "Checking non-object permission [%s] for user %d at org %d",
2211                                         perm, userid, atoi(context_org) );
2212                         result = dbi_conn_queryf(
2213                                 writehandle,
2214                                 "SELECT permission.usr_has_perm(%d, '%s', %d) AS has_perm;",
2215                                 userid,
2216                                 perm,
2217                                 atoi( context_org )
2218                         );
2219
2220                         if( result ) {
2221                                 osrfLogDebug( OSRF_LOG_MARK,
2222                                         "Received a result for permission [%s] for user %d at org %d",
2223                                         perm, userid, atoi( context_org ));
2224                                 if( dbi_result_first_row( result )) {
2225                                         jsonObject* return_val = oilsMakeJSONFromResult( result );
2226                                         const char* has_perm = jsonObjectGetString(
2227                                                 jsonObjectGetKeyConst( return_val, "has_perm" ));
2228                                         osrfLogDebug( OSRF_LOG_MARK,
2229                                                 "Status of permission [%s] for user %d at org %d is [%s]",
2230                                                 perm, userid, atoi( context_org ), has_perm );
2231                                         if( *has_perm == 't' )
2232                                                 OK = 1;
2233                                         jsonObjectFree( return_val );
2234                                 }
2235
2236                                 dbi_result_free( result );
2237                                 if( OK )
2238                                         break;
2239                         } else {
2240                                 const char* msg;
2241                                 int errnum = dbi_conn_error( writehandle, &msg );
2242                                 osrfLogWarning( OSRF_LOG_MARK, "Unable to call user object permissions: %d, %s",
2243                                         errnum, msg ? msg : "(No description available)" );
2244                                 if( !oilsIsDBConnected( writehandle ))
2245                                         osrfAppSessionPanic( ctx->session );
2246                         }
2247
2248                 }
2249
2250                 if( OK )
2251                         break;
2252         }
2253
2254         osrfStringArrayFree( context_org_array );
2255
2256         return OK;
2257 }
2258
2259 /**
2260         @brief Look up the root of the org_unit tree.
2261         @param ctx Pointer to the method context.
2262         @return The id of the root org unit, as a character string.
2263
2264         Query actor.org_unit where parent_ou is null, and return the id as a string.
2265
2266         This function assumes that there is only one root org unit, i.e. that we
2267         have a single tree, not a forest.
2268
2269         The calling code is responsible for freeing the returned string.
2270 */
2271 static const char* org_tree_root( osrfMethodContext* ctx ) {
2272
2273         static char cached_root_id[ 32 ] = "";  // extravagantly large buffer
2274         static time_t last_lookup_time = 0;
2275         time_t current_time = time( NULL );
2276
2277         if( cached_root_id[ 0 ] && ( current_time - last_lookup_time < 3600 ) ) {
2278                 // We successfully looked this up less than an hour ago.
2279                 // It's not likely to have changed since then.
2280                 return strdup( cached_root_id );
2281         }
2282         last_lookup_time = current_time;
2283
2284         int err = 0;
2285         jsonObject* where_clause = single_hash( "parent_ou", NULL );
2286         jsonObject* result = doFieldmapperSearch(
2287                 ctx, osrfHashGet( oilsIDL(), "aou" ), where_clause, NULL, &err );
2288         jsonObjectFree( where_clause );
2289
2290         jsonObject* tree_top = jsonObjectGetIndex( result, 0 );
2291
2292         if( !tree_top ) {
2293                 jsonObjectFree( result );
2294
2295                 growing_buffer* msg = buffer_init( 128 );
2296                 OSRF_BUFFER_ADD( msg, modulename );
2297                 OSRF_BUFFER_ADD( msg,
2298                                 ": Internal error, could not find the top of the org tree (parent_ou = NULL)" );
2299
2300                 char* m = buffer_release( msg );
2301                 osrfAppSessionStatus( ctx->session,
2302                                 OSRF_STATUS_INTERNALSERVERERROR, "osrfMethodException", ctx->request, m );
2303                 free( m );
2304
2305                 cached_root_id[ 0 ] = '\0';
2306                 return NULL;
2307         }
2308
2309         const char* root_org_unit_id = oilsFMGetStringConst( tree_top, "id" );
2310         osrfLogDebug( OSRF_LOG_MARK, "Top of the org tree is %s", root_org_unit_id );
2311
2312         strcpy( cached_root_id, root_org_unit_id );
2313         jsonObjectFree( result );
2314         return cached_root_id;
2315 }
2316
2317 /**
2318         @brief Create a JSON_HASH with a single key/value pair.
2319         @param key The key of the key/value pair.
2320         @param value the value of the key/value pair.
2321         @return Pointer to a newly created jsonObject of type JSON_HASH.
2322
2323         The value of the key/value is either a string or (if @a value is NULL) a null.
2324 */
2325 static jsonObject* single_hash( const char* key, const char* value ) {
2326         // Sanity check
2327         if( ! key ) key = "";
2328
2329         jsonObject* hash = jsonNewObjectType( JSON_HASH );
2330         jsonObjectSetKey( hash, key, jsonNewObject( value ) );
2331         return hash;
2332 }
2333
2334
2335 int doCreate( osrfMethodContext* ctx ) {
2336         if(osrfMethodVerifyContext( ctx )) {
2337                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2338                 return -1;
2339         }
2340
2341         if( enforce_pcrud )
2342                 timeout_needs_resetting = 1;
2343
2344         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2345         jsonObject* target = NULL;
2346         jsonObject* options = NULL;
2347
2348         if( enforce_pcrud ) {
2349                 target = jsonObjectGetIndex( ctx->params, 1 );
2350                 options = jsonObjectGetIndex( ctx->params, 2 );
2351         } else {
2352                 target = jsonObjectGetIndex( ctx->params, 0 );
2353                 options = jsonObjectGetIndex( ctx->params, 1 );
2354         }
2355
2356         if( !verifyObjectClass( ctx, target )) {
2357                 osrfAppRespondComplete( ctx, NULL );
2358                 return -1;
2359         }
2360
2361         osrfLogDebug( OSRF_LOG_MARK, "Object seems to be of the correct type" );
2362
2363         const char* trans_id = getXactId( ctx );
2364         if( !trans_id ) {
2365                 osrfLogError( OSRF_LOG_MARK, "No active transaction -- required for CREATE" );
2366
2367                 osrfAppSessionStatus(
2368                         ctx->session,
2369                         OSRF_STATUS_BADREQUEST,
2370                         "osrfMethodException",
2371                         ctx->request,
2372                         "No active transaction -- required for CREATE"
2373                 );
2374                 osrfAppRespondComplete( ctx, NULL );
2375                 return -1;
2376         }
2377
2378         // The following test is harmless but redundant.  If a class is
2379         // readonly, we don't register a create method for it.
2380         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
2381                 osrfAppSessionStatus(
2382                         ctx->session,
2383                         OSRF_STATUS_BADREQUEST,
2384                         "osrfMethodException",
2385                         ctx->request,
2386                         "Cannot INSERT readonly class"
2387                 );
2388                 osrfAppRespondComplete( ctx, NULL );
2389                 return -1;
2390         }
2391
2392         // Set the last_xact_id
2393         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
2394         if( index > -1 ) {
2395                 osrfLogDebug(OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
2396                         trans_id, target->classname, index);
2397                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
2398         }
2399
2400         osrfLogDebug( OSRF_LOG_MARK, "There is a transaction running..." );
2401
2402         dbhandle = writehandle;
2403
2404         osrfHash* fields = osrfHashGet( meta, "fields" );
2405         char* pkey       = osrfHashGet( meta, "primarykey" );
2406         char* seq        = osrfHashGet( meta, "sequence" );
2407
2408         growing_buffer* table_buf = buffer_init( 128 );
2409         growing_buffer* col_buf   = buffer_init( 128 );
2410         growing_buffer* val_buf   = buffer_init( 128 );
2411
2412         OSRF_BUFFER_ADD( table_buf, "INSERT INTO " );
2413         OSRF_BUFFER_ADD( table_buf, osrfHashGet( meta, "tablename" ));
2414         OSRF_BUFFER_ADD_CHAR( col_buf, '(' );
2415         buffer_add( val_buf,"VALUES (" );
2416
2417
2418         int first = 1;
2419         osrfHash* field = NULL;
2420         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
2421         while( (field = osrfHashIteratorNext( field_itr ) ) ) {
2422
2423                 const char* field_name = osrfHashIteratorKey( field_itr );
2424
2425                 if( str_is_true( osrfHashGet( field, "virtual" ) ) )
2426                         continue;
2427
2428                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
2429
2430                 char* value;
2431                 if( field_object && field_object->classname ) {
2432                         value = oilsFMGetString(
2433                                 field_object,
2434                                 (char*)oilsIDLFindPath( "/%s/primarykey", field_object->classname )
2435                         );
2436                 } else if( field_object && JSON_BOOL == field_object->type ) {
2437                         if( jsonBoolIsTrue( field_object ) )
2438                                 value = strdup( "t" );
2439                         else
2440                                 value = strdup( "f" );
2441                 } else {
2442                         value = jsonObjectToSimpleString( field_object );
2443                 }
2444
2445                 if( first ) {
2446                         first = 0;
2447                 } else {
2448                         OSRF_BUFFER_ADD_CHAR( col_buf, ',' );
2449                         OSRF_BUFFER_ADD_CHAR( val_buf, ',' );
2450                 }
2451
2452                 buffer_add( col_buf, field_name );
2453
2454                 if( !field_object || field_object->type == JSON_NULL ) {
2455                         buffer_add( val_buf, "DEFAULT" );
2456
2457                 } else if( !strcmp( get_primitive( field ), "number" )) {
2458                         const char* numtype = get_datatype( field );
2459                         if( !strcmp( numtype, "INT8" )) {
2460                                 buffer_fadd( val_buf, "%lld", atoll( value ));
2461
2462                         } else if( !strcmp( numtype, "INT" )) {
2463                                 buffer_fadd( val_buf, "%d", atoi( value ));
2464
2465                         } else if( !strcmp( numtype, "NUMERIC" )) {
2466                                 buffer_fadd( val_buf, "%f", atof( value ));
2467                         }
2468                 } else {
2469                         if( dbi_conn_quote_string( writehandle, &value )) {
2470                                 OSRF_BUFFER_ADD( val_buf, value );
2471
2472                         } else {
2473                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
2474                                 osrfAppSessionStatus(
2475                                         ctx->session,
2476                                         OSRF_STATUS_INTERNALSERVERERROR,
2477                                         "osrfMethodException",
2478                                         ctx->request,
2479                                         "Error quoting string -- please see the error log for more details"
2480                                 );
2481                                 free( value );
2482                                 buffer_free( table_buf );
2483                                 buffer_free( col_buf );
2484                                 buffer_free( val_buf );
2485                                 osrfAppRespondComplete( ctx, NULL );
2486                                 return -1;
2487                         }
2488                 }
2489
2490                 free( value );
2491         }
2492
2493         osrfHashIteratorFree( field_itr );
2494
2495         OSRF_BUFFER_ADD_CHAR( col_buf, ')' );
2496         OSRF_BUFFER_ADD_CHAR( val_buf, ')' );
2497
2498         char* table_str = buffer_release( table_buf );
2499         char* col_str   = buffer_release( col_buf );
2500         char* val_str   = buffer_release( val_buf );
2501         growing_buffer* sql = buffer_init( 128 );
2502         buffer_fadd( sql, "%s %s %s;", table_str, col_str, val_str );
2503         free( table_str );
2504         free( col_str );
2505         free( val_str );
2506
2507         char* query = buffer_release( sql );
2508
2509         osrfLogDebug( OSRF_LOG_MARK, "%s: Insert SQL [%s]", modulename, query );
2510
2511         jsonObject* obj = NULL;
2512         int rc = 0;
2513
2514         dbi_result result = dbi_conn_query( writehandle, query );
2515         if( !result ) {
2516                 obj = jsonNewObject( NULL );
2517                 const char* msg;
2518                 int errnum = dbi_conn_error( writehandle, &msg );
2519                 osrfLogError(
2520                         OSRF_LOG_MARK,
2521                         "%s ERROR inserting %s object using query [%s]: %d %s",
2522                         modulename,
2523                         osrfHashGet(meta, "fieldmapper"),
2524                         query,
2525                         errnum,
2526                         msg ? msg : "(No description available)"
2527                 );
2528                 osrfAppSessionStatus(
2529                         ctx->session,
2530                         OSRF_STATUS_INTERNALSERVERERROR,
2531                         "osrfMethodException",
2532                         ctx->request,
2533                         "INSERT error -- please see the error log for more details"
2534                 );
2535                 if( !oilsIsDBConnected( writehandle ))
2536                         osrfAppSessionPanic( ctx->session );
2537                 rc = -1;
2538         } else {
2539                 dbi_result_free( result );
2540
2541                 char* id = oilsFMGetString( target, pkey );
2542                 if( !id ) {
2543                         unsigned long long new_id = dbi_conn_sequence_last( writehandle, seq );
2544                         growing_buffer* _id = buffer_init( 10 );
2545                         buffer_fadd( _id, "%lld", new_id );
2546                         id = buffer_release( _id );
2547                 }
2548
2549                 // Find quietness specification, if present
2550                 const char* quiet_str = NULL;
2551                 if( options ) {
2552                         const jsonObject* quiet_obj = jsonObjectGetKeyConst( options, "quiet" );
2553                         if( quiet_obj )
2554                                 quiet_str = jsonObjectGetString( quiet_obj );
2555                 }
2556
2557                 if( str_is_true( quiet_str )) {  // if quietness is specified
2558                         obj = jsonNewObject( id );
2559                 }
2560                 else {
2561
2562                         // Fetch the row that we just inserted, so that we can return it to the client
2563                         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2564                         jsonObjectSetKey( where_clause, pkey, jsonNewObject( id ));
2565
2566                         int err = 0;
2567                         jsonObject* list = doFieldmapperSearch( ctx, meta, where_clause, NULL, &err );
2568                         if( err )
2569                                 rc = -1;
2570                         else
2571                                 obj = jsonObjectClone( jsonObjectGetIndex( list, 0 ));
2572
2573                         jsonObjectFree( list );
2574                         jsonObjectFree( where_clause );
2575                 }
2576
2577                 free( id );
2578         }
2579
2580         free( query );
2581         osrfAppRespondComplete( ctx, obj );
2582         jsonObjectFree( obj );
2583         return rc;
2584 }
2585
2586 /**
2587         @brief Implement the retrieve method.
2588         @param ctx Pointer to the method context.
2589         @param err Pointer through which to return an error code.
2590         @return If successful, a pointer to the result to be returned to the client;
2591         otherwise NULL.
2592
2593         From the method's class, fetch a row with a specified value in the primary key.  This
2594         method relies on the database design convention that a primary key consists of a single
2595         column.
2596
2597         Method parameters:
2598         - authkey (PCRUD only)
2599         - value of the primary key for the desired row, for building the WHERE clause
2600         - a JSON_HASH containing any other SQL clauses: select, join, etc.
2601
2602         Return to client: One row from the query.
2603 */
2604 int doRetrieve( osrfMethodContext* ctx ) {
2605         if(osrfMethodVerifyContext( ctx )) {
2606                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
2607                 return -1;
2608         }
2609
2610         if( enforce_pcrud )
2611                 timeout_needs_resetting = 1;
2612
2613         int id_pos = 0;
2614         int order_pos = 1;
2615
2616         if( enforce_pcrud ) {
2617                 id_pos = 1;
2618                 order_pos = 2;
2619         }
2620
2621         // Get the class metadata
2622         osrfHash* class_def = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
2623
2624         // Get the value of the primary key, from a method parameter
2625         const jsonObject* id_obj = jsonObjectGetIndex( ctx->params, id_pos );
2626
2627         osrfLogDebug(
2628                 OSRF_LOG_MARK,
2629                 "%s retrieving %s object with primary key value of %s",
2630                 modulename,
2631                 osrfHashGet( class_def, "fieldmapper" ),
2632                 jsonObjectGetString( id_obj )
2633         );
2634
2635         // Build a WHERE clause based on the key value
2636         jsonObject* where_clause = jsonNewObjectType( JSON_HASH );
2637         jsonObjectSetKey(
2638                 where_clause,
2639                 osrfHashGet( class_def, "primarykey" ),  // name of key column
2640                 jsonObjectClone( id_obj )                // value of key column
2641         );
2642
2643         jsonObject* rest_of_query = jsonObjectGetIndex( ctx->params, order_pos );
2644
2645         // Do the query
2646         int err = 0;
2647         jsonObject* list = doFieldmapperSearch( ctx, class_def, where_clause, rest_of_query, &err );
2648
2649         jsonObjectFree( where_clause );
2650         if( err ) {
2651                 osrfAppRespondComplete( ctx, NULL );
2652                 return -1;
2653         }
2654
2655         jsonObject* obj = jsonObjectExtractIndex( list, 0 );
2656         jsonObjectFree( list );
2657
2658         if( enforce_pcrud ) {
2659                 // no result, skip this entirely
2660                 if(NULL != obj && !verifyObjectPCRUD( ctx, class_def, obj, 1 )) {
2661                         jsonObjectFree( obj );
2662
2663                         growing_buffer* msg = buffer_init( 128 );
2664                         OSRF_BUFFER_ADD( msg, modulename );
2665                         OSRF_BUFFER_ADD( msg, ": Insufficient permissions to retrieve object" );
2666
2667                         char* m = buffer_release( msg );
2668                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_NOTALLOWED, "osrfMethodException",
2669                                         ctx->request, m );
2670                         free( m );
2671
2672                         osrfAppRespondComplete( ctx, NULL );
2673                         return -1;
2674                 }
2675         }
2676
2677         // doFieldmapperSearch() now does the responding for us
2678         //osrfAppRespondComplete( ctx, obj );
2679         osrfAppRespondComplete( ctx, NULL );
2680
2681         jsonObjectFree( obj );
2682         return 0;
2683 }
2684
2685 /**
2686         @brief Translate a numeric value to a string representation for the database.
2687         @param field Pointer to the IDL field definition.
2688         @param value Pointer to a jsonObject holding the value of a field.
2689         @return Pointer to a newly allocated string.
2690
2691         The input object is typically a JSON_NUMBER, but it may be a JSON_STRING as long as
2692         its contents are numeric.  A non-numeric string is likely to result in invalid SQL.
2693
2694         If the datatype of the receiving field is not numeric, wrap the value in quotes.
2695
2696         The calling code is responsible for freeing the resulting string by calling free().
2697 */
2698 static char* jsonNumberToDBString( osrfHash* field, const jsonObject* value ) {
2699         growing_buffer* val_buf = buffer_init( 32 );
2700
2701         // If the value is a number and the DB field is numeric, no quotes needed
2702         if( value->type == JSON_NUMBER && !strcmp( get_primitive( field ), "number") ) {
2703                 buffer_fadd( val_buf, jsonObjectGetString( value ) );
2704         } else {
2705                 // Presumably this was really intended to be a string, so quote it
2706                 char* str = jsonObjectToSimpleString( value );
2707                 if( dbi_conn_quote_string( dbhandle, &str )) {
2708                         OSRF_BUFFER_ADD( val_buf, str );
2709                         free( str );
2710                 } else {
2711                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]", modulename, str );
2712                         free( str );
2713                         buffer_free( val_buf );
2714                         return NULL;
2715                 }
2716         }
2717
2718         return buffer_release( val_buf );
2719 }
2720
2721 static char* searchINPredicate( const char* class_alias, osrfHash* field,
2722                 jsonObject* node, const char* op, osrfMethodContext* ctx ) {
2723         growing_buffer* sql_buf = buffer_init( 32 );
2724
2725         buffer_fadd(
2726                 sql_buf,
2727                 "\"%s\".%s ",
2728                 class_alias,
2729                 osrfHashGet( field, "name" )
2730         );
2731
2732         if( !op ) {
2733                 buffer_add( sql_buf, "IN (" );
2734         } else if( !strcasecmp( op,"not in" )) {
2735                 buffer_add( sql_buf, "NOT IN (" );
2736         } else {
2737                 buffer_add( sql_buf, "IN (" );
2738         }
2739
2740         if( node->type == JSON_HASH ) {
2741                 // subquery predicate
2742                 char* subpred = buildQuery( ctx, node, SUBSELECT );
2743                 if( ! subpred ) {
2744                         buffer_free( sql_buf );
2745                         return NULL;
2746                 }
2747
2748                 buffer_add( sql_buf, subpred );
2749                 free( subpred );
2750
2751         } else if( node->type == JSON_ARRAY ) {
2752                 // literal value list
2753                 int in_item_index = 0;
2754                 int in_item_first = 1;
2755                 const jsonObject* in_item;
2756                 while( (in_item = jsonObjectGetIndex( node, in_item_index++ )) ) {
2757
2758                         if( in_item_first )
2759                                 in_item_first = 0;
2760                         else
2761                                 buffer_add( sql_buf, ", " );
2762
2763                         // Sanity check
2764                         if( in_item->type != JSON_STRING && in_item->type != JSON_NUMBER ) {
2765                                 osrfLogError( OSRF_LOG_MARK,
2766                                                 "%s: Expected string or number within IN list; found %s",
2767                                                 modulename, json_type( in_item->type ) );
2768                                 buffer_free( sql_buf );
2769                                 return NULL;
2770                         }
2771
2772                         // Append the literal value -- quoted if not a number
2773                         if( JSON_NUMBER == in_item->type ) {
2774                                 char* val = jsonNumberToDBString( field, in_item );
2775                                 OSRF_BUFFER_ADD( sql_buf, val );
2776                                 free( val );
2777
2778                         } else if( !strcmp( get_primitive( field ), "number" )) {
2779                                 char* val = jsonNumberToDBString( field, in_item );
2780                                 OSRF_BUFFER_ADD( sql_buf, val );
2781                                 free( val );
2782
2783                         } else {
2784                                 char* key_string = jsonObjectToSimpleString( in_item );
2785                                 if( dbi_conn_quote_string( dbhandle, &key_string )) {
2786                                         OSRF_BUFFER_ADD( sql_buf, key_string );
2787                                         free( key_string );
2788                                 } else {
2789                                         osrfLogError( OSRF_LOG_MARK,
2790                                                         "%s: Error quoting key string [%s]", modulename, key_string );
2791                                         free( key_string );
2792                                         buffer_free( sql_buf );
2793                                         return NULL;
2794                                 }
2795                         }
2796                 }
2797
2798                 if( in_item_first ) {
2799                         osrfLogError(OSRF_LOG_MARK, "%s: Empty IN list", modulename );
2800                         buffer_free( sql_buf );
2801                         return NULL;
2802                 }
2803         } else {
2804                 osrfLogError( OSRF_LOG_MARK, "%s: Expected object or array for IN clause; found %s",
2805                         modulename, json_type( node->type ));
2806                 buffer_free( sql_buf );
2807                 return NULL;
2808         }
2809
2810         OSRF_BUFFER_ADD_CHAR( sql_buf, ')' );
2811
2812         return buffer_release( sql_buf );
2813 }
2814
2815 // Receive a JSON_ARRAY representing a function call.  The first
2816 // entry in the array is the function name.  The rest are parameters.
2817 static char* searchValueTransform( const jsonObject* array ) {
2818
2819         if( array->size < 1 ) {
2820                 osrfLogError( OSRF_LOG_MARK, "%s: Empty array for value transform", modulename );
2821                 return NULL;
2822         }
2823
2824         // Get the function name
2825         jsonObject* func_item = jsonObjectGetIndex( array, 0 );
2826         if( func_item->type != JSON_STRING ) {
2827                 osrfLogError( OSRF_LOG_MARK, "%s: Error: expected function name, found %s",
2828                         modulename, json_type( func_item->type ));
2829                 return NULL;
2830         }
2831
2832         growing_buffer* sql_buf = buffer_init( 32 );
2833
2834         OSRF_BUFFER_ADD( sql_buf, jsonObjectGetString( func_item ) );
2835         OSRF_BUFFER_ADD( sql_buf, "( " );
2836
2837         // Get the parameters
2838         int func_item_index = 1;   // We already grabbed the zeroth entry
2839         while( (func_item = jsonObjectGetIndex( array, func_item_index++ )) ) {
2840
2841                 // Add a separator comma, if we need one
2842                 if( func_item_index > 2 )
2843                         buffer_add( sql_buf, ", " );
2844
2845                 // Add the current parameter
2846                 if( func_item->type == JSON_NULL ) {
2847                         buffer_add( sql_buf, "NULL" );
2848                 } else {
2849                         if( func_item->type == JSON_BOOL ) {
2850                                 if( jsonBoolIsTrue(func_item) ) {
2851                                         buffer_add( sql_buf, "TRUE" );
2852                                 } else {
2853                                         buffer_add( sql_buf, "FALSE" );
2854                                 }
2855                         } else {
2856                                 char* val = jsonObjectToSimpleString( func_item );
2857                                 if( dbi_conn_quote_string( dbhandle, &val )) {
2858                                         OSRF_BUFFER_ADD( sql_buf, val );
2859                                         free( val );
2860                                 } else {
2861                                         osrfLogError( OSRF_LOG_MARK,
2862                                                 "%s: Error quoting key string [%s]", modulename, val );
2863                                         buffer_free( sql_buf );
2864                                         free( val );
2865                                         return NULL;
2866                                 }
2867                         }
2868                 }
2869         }
2870
2871         buffer_add( sql_buf, " )" );
2872
2873         return buffer_release( sql_buf );
2874 }
2875
2876 static char* searchFunctionPredicate( const char* class_alias, osrfHash* field,
2877                 const jsonObject* node, const char* op ) {
2878
2879         if( ! is_good_operator( op ) ) {
2880                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
2881                 return NULL;
2882         }
2883
2884         char* val = searchValueTransform( node );
2885         if( !val )
2886                 return NULL;
2887
2888         const char* right_percent = "";
2889         const char* real_op       = op;
2890
2891         if( !strcasecmp( op, "startwith") ) {
2892                 real_op = "like";
2893                 right_percent = "|| '%'";
2894         }
2895
2896         growing_buffer* sql_buf = buffer_init( 32 );
2897         buffer_fadd(
2898                 sql_buf,
2899                 "\"%s\".%s %s %s%s",
2900                 class_alias,
2901                 osrfHashGet( field, "name" ),
2902                 real_op,
2903                 val,
2904                 right_percent
2905         );
2906
2907         free( val );
2908
2909         return buffer_release( sql_buf );
2910 }
2911
2912 // class_alias is a class name or other table alias
2913 // field is a field definition as stored in the IDL
2914 // node comes from the method parameter, and may represent an entry in the SELECT list
2915 static char* searchFieldTransform( const char* class_alias, osrfHash* field,
2916                 const jsonObject* node ) {
2917         growing_buffer* sql_buf = buffer_init( 32 );
2918
2919         const char* field_transform = jsonObjectGetString(
2920                 jsonObjectGetKeyConst( node, "transform" ) );
2921         const char* transform_subcolumn = jsonObjectGetString(
2922                 jsonObjectGetKeyConst( node, "result_field" ) );
2923
2924         if( transform_subcolumn ) {
2925                 if( ! is_identifier( transform_subcolumn ) ) {
2926                         osrfLogError( OSRF_LOG_MARK, "%s: Invalid subfield name: \"%s\"\n",
2927                                         modulename, transform_subcolumn );
2928                         buffer_free( sql_buf );
2929                         return NULL;
2930                 }
2931                 OSRF_BUFFER_ADD_CHAR( sql_buf, '(' );    // enclose transform in parentheses
2932         }
2933
2934         if( field_transform ) {
2935
2936                 if( ! is_identifier( field_transform ) ) {
2937                         osrfLogError( OSRF_LOG_MARK, "%s: Expected function name, found \"%s\"\n",
2938                                         modulename, field_transform );
2939                         buffer_free( sql_buf );
2940                         return NULL;
2941                 }
2942
2943                 if( obj_is_true( jsonObjectGetKeyConst( node, "distinct" ) ) ) {
2944                         buffer_fadd( sql_buf, "%s(DISTINCT \"%s\".%s",
2945                                 field_transform, class_alias, osrfHashGet( field, "name" ));
2946                 } else {
2947                         buffer_fadd( sql_buf, "%s(\"%s\".%s",
2948                                 field_transform, class_alias, osrfHashGet( field, "name" ));
2949                 }
2950
2951                 const jsonObject* array = jsonObjectGetKeyConst( node, "params" );
2952
2953                 if( array ) {
2954                         if( array->type != JSON_ARRAY ) {
2955                                 osrfLogError( OSRF_LOG_MARK,
2956                                         "%s: Expected JSON_ARRAY for function params; found %s",
2957                                         modulename, json_type( array->type ) );
2958                                 buffer_free( sql_buf );
2959                                 return NULL;
2960                         }
2961                         int func_item_index = 0;
2962                         jsonObject* func_item;
2963                         while( (func_item = jsonObjectGetIndex( array, func_item_index++ ))) {
2964
2965                                 char* val = jsonObjectToSimpleString( func_item );
2966
2967                                 if( !val ) {
2968                                         buffer_add( sql_buf, ",NULL" );
2969                                 } else if( dbi_conn_quote_string( dbhandle, &val )) {
2970                                         OSRF_BUFFER_ADD_CHAR( sql_buf, ',' );
2971                                         OSRF_BUFFER_ADD( sql_buf, val );
2972                                 } else {
2973                                         osrfLogError( OSRF_LOG_MARK,
2974                                                         "%s: Error quoting key string [%s]", modulename, val );
2975                                         free( val );
2976                                         buffer_free( sql_buf );
2977                                         return NULL;
2978                                 }
2979                                 free( val );
2980                         }
2981                 }
2982
2983                 buffer_add( sql_buf, " )" );
2984
2985         } else {
2986                 buffer_fadd( sql_buf, "\"%s\".%s", class_alias, osrfHashGet( field, "name" ));
2987         }
2988
2989         if( transform_subcolumn )
2990                 buffer_fadd( sql_buf, ").\"%s\"", transform_subcolumn );
2991
2992         return buffer_release( sql_buf );
2993 }
2994
2995 static char* searchFieldTransformPredicate( const ClassInfo* class_info, osrfHash* field,
2996                 const jsonObject* node, const char* op ) {
2997
2998         if( ! is_good_operator( op ) ) {
2999                 osrfLogError( OSRF_LOG_MARK, "%s: Error: Invalid operator %s", modulename, op );
3000                 return NULL;
3001         }
3002
3003         char* field_transform = searchFieldTransform( class_info->alias, field, node );
3004         if( ! field_transform )
3005                 return NULL;
3006         char* value = NULL;
3007         int extra_parens = 0;   // boolean
3008
3009         const jsonObject* value_obj = jsonObjectGetKeyConst( node, "value" );
3010         if( ! value_obj ) {
3011                 value = searchWHERE( node, class_info, AND_OP_JOIN, NULL );
3012                 if( !value ) {
3013                         osrfLogError( OSRF_LOG_MARK, "%s: Error building condition for field transform",
3014                                 modulename );
3015                         free( field_transform );
3016                         return NULL;
3017                 }
3018                 extra_parens = 1;
3019         } else if( value_obj->type == JSON_ARRAY ) {
3020                 value = searchValueTransform( value_obj );
3021                 if( !value ) {
3022                         osrfLogError( OSRF_LOG_MARK,
3023                                 "%s: Error building value transform for field transform", modulename );
3024                         free( field_transform );
3025                         return NULL;
3026                 }
3027         } else if( value_obj->type == JSON_HASH ) {
3028                 value = searchWHERE( value_obj, class_info, AND_OP_JOIN, NULL );
3029                 if( !value ) {
3030                         osrfLogError( OSRF_LOG_MARK, "%s: Error building predicate for field transform",
3031                                 modulename );
3032                         free( field_transform );
3033                         return NULL;
3034                 }
3035                 extra_parens = 1;
3036         } else if( value_obj->type == JSON_NUMBER ) {
3037                 value = jsonNumberToDBString( field, value_obj );
3038         } else if( value_obj->type == JSON_NULL ) {
3039                 osrfLogError( OSRF_LOG_MARK,
3040                         "%s: Error building predicate for field transform: null value", modulename );
3041                 free( field_transform );
3042                 return NULL;
3043         } else if( value_obj->type == JSON_BOOL ) {
3044                 osrfLogError( OSRF_LOG_MARK,
3045                         "%s: Error building predicate for field transform: boolean value", modulename );
3046                 free( field_transform );
3047                 return NULL;
3048         } else {
3049                 if( !strcmp( get_primitive( field ), "number") ) {
3050                         value = jsonNumberToDBString( field, value_obj );
3051                 } else {
3052                         value = jsonObjectToSimpleString( value_obj );
3053                         if( !dbi_conn_quote_string( dbhandle, &value )) {
3054                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3055                                         modulename, value );
3056                                 free( value );
3057                                 free( field_transform );
3058                                 return NULL;
3059                         }
3060                 }
3061         }
3062
3063         const char* left_parens  = "";
3064         const char* right_parens = "";
3065
3066         if( extra_parens ) {
3067                 left_parens  = "(";
3068                 right_parens = ")";
3069         }
3070
3071         const char* right_percent = "";
3072         const char* real_op       = op;
3073
3074         if( !strcasecmp( op, "startwith") ) {
3075                 real_op = "like";
3076                 right_percent = "|| '%'";
3077         }
3078
3079         growing_buffer* sql_buf = buffer_init( 32 );
3080
3081         buffer_fadd(
3082                 sql_buf,
3083                 "%s%s %s %s %s%s %s%s",
3084                 left_parens,
3085                 field_transform,
3086                 real_op,
3087                 left_parens,
3088                 value,
3089                 right_percent,
3090                 right_parens,
3091                 right_parens
3092         );
3093
3094         free( value );
3095         free( field_transform );
3096
3097         return buffer_release( sql_buf );
3098 }
3099
3100 static char* searchSimplePredicate( const char* op, const char* class_alias,
3101                 osrfHash* field, const jsonObject* node ) {
3102
3103         if( ! is_good_operator( op ) ) {
3104                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid operator [%s]", modulename, op );
3105                 return NULL;
3106         }
3107
3108         char* val = NULL;
3109
3110         // Get the value to which we are comparing the specified column
3111         if( node->type != JSON_NULL ) {
3112                 if( node->type == JSON_NUMBER ) {
3113                         val = jsonNumberToDBString( field, node );
3114                 } else if( !strcmp( get_primitive( field ), "number" ) ) {
3115                         val = jsonNumberToDBString( field, node );
3116                 } else {
3117                         val = jsonObjectToSimpleString( node );
3118                 }
3119         }
3120
3121         if( val ) {
3122                 if( JSON_NUMBER != node->type && strcmp( get_primitive( field ), "number") ) {
3123                         // Value is not numeric; enclose it in quotes
3124                         if( !dbi_conn_quote_string( dbhandle, &val ) ) {
3125                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key string [%s]",
3126                                         modulename, val );
3127                                 free( val );
3128                                 return NULL;
3129                         }
3130                 }
3131         } else {
3132                 // Compare to a null value
3133                 val = strdup( "NULL" );
3134                 if( strcmp( op, "=" ))
3135                         op = "IS NOT";
3136                 else
3137                         op = "IS";
3138         }
3139
3140         const char* right_percent = "";
3141         const char* real_op       = op;
3142
3143         if( !strcasecmp( op, "startwith") ) {
3144                 real_op = "like";
3145                 right_percent = "|| '%'";
3146         }
3147
3148         growing_buffer* sql_buf = buffer_init( 32 );
3149         buffer_fadd( sql_buf, "\"%s\".%s %s %s%s", class_alias, osrfHashGet(field, "name"), real_op, val, right_percent );
3150         char* pred = buffer_release( sql_buf );
3151
3152         free( val );
3153
3154         return pred;
3155 }
3156
3157 static char* searchBETWEENPredicate( const char* class_alias,
3158                 osrfHash* field, const jsonObject* node ) {
3159
3160         const jsonObject* x_node = jsonObjectGetIndex( node, 0 );
3161         const jsonObject* y_node = jsonObjectGetIndex( node, 1 );
3162
3163         if( NULL == y_node ) {
3164                 osrfLogError( OSRF_LOG_MARK, "%s: Not enough operands for BETWEEN operator", modulename );
3165                 return NULL;
3166         }
3167         else if( NULL != jsonObjectGetIndex( node, 2 ) ) {
3168                 osrfLogError( OSRF_LOG_MARK, "%s: Too many operands for BETWEEN operator", modulename );
3169                 return NULL;
3170         }
3171
3172         char* x_string;
3173         char* y_string;
3174
3175         if( !strcmp( get_primitive( field ), "number") ) {
3176                 x_string = jsonNumberToDBString( field, x_node );
3177                 y_string = jsonNumberToDBString( field, y_node );
3178
3179         } else {
3180                 x_string = jsonObjectToSimpleString( x_node );
3181                 y_string = jsonObjectToSimpleString( y_node );
3182                 if( !(dbi_conn_quote_string( dbhandle, &x_string )
3183                         && dbi_conn_quote_string( dbhandle, &y_string )) ) {
3184                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting key strings [%s] and [%s]",
3185                                         modulename, x_string, y_string );
3186                         free( x_string );
3187                         free( y_string );
3188                         return NULL;
3189                 }
3190         }
3191
3192         growing_buffer* sql_buf = buffer_init( 32 );
3193         buffer_fadd( sql_buf, "\"%s\".%s BETWEEN %s AND %s",
3194                         class_alias, osrfHashGet( field, "name" ), x_string, y_string );
3195         free( x_string );
3196         free( y_string );
3197
3198         return buffer_release( sql_buf );
3199 }
3200
3201 static char* searchPredicate( const ClassInfo* class_info, osrfHash* field,
3202                                                           jsonObject* node, osrfMethodContext* ctx ) {
3203
3204         char* pred = NULL;
3205         if( node->type == JSON_ARRAY ) { // equality IN search
3206                 pred = searchINPredicate( class_info->alias, field, node, NULL, ctx );
3207         } else if( node->type == JSON_HASH ) { // other search
3208                 jsonIterator* pred_itr = jsonNewIterator( node );
3209                 if( !jsonIteratorHasNext( pred_itr ) ) {
3210                         osrfLogError( OSRF_LOG_MARK, "%s: Empty predicate for field \"%s\"",
3211                                         modulename, osrfHashGet(field, "name" ));
3212                 } else {
3213                         jsonObject* pred_node = jsonIteratorNext( pred_itr );
3214
3215                         // Verify that there are no additional predicates
3216                         if( jsonIteratorHasNext( pred_itr ) ) {
3217                                 osrfLogError( OSRF_LOG_MARK, "%s: Multiple predicates for field \"%s\"",
3218                                                 modulename, osrfHashGet(field, "name" ));
3219                         } else if( !(strcasecmp( pred_itr->key,"between" )) )
3220                                 pred = searchBETWEENPredicate( class_info->alias, field, pred_node );
3221                         else if( !(strcasecmp( pred_itr->key,"in" ))
3222                                         || !(strcasecmp( pred_itr->key,"not in" )) )
3223                                 pred = searchINPredicate(
3224                                         class_info->alias, field, pred_node, pred_itr->key, ctx );
3225                         else if( pred_node->type == JSON_ARRAY )
3226                                 pred = searchFunctionPredicate(
3227                                         class_info->alias, field, pred_node, pred_itr->key );
3228                         else if( pred_node->type == JSON_HASH )
3229                                 pred = searchFieldTransformPredicate(
3230                                         class_info, field, pred_node, pred_itr->key );
3231                         else
3232                                 pred = searchSimplePredicate( pred_itr->key, class_info->alias, field, pred_node );
3233                 }
3234                 jsonIteratorFree( pred_itr );
3235
3236         } else if( node->type == JSON_NULL ) { // IS NULL search
3237                 growing_buffer* _p = buffer_init( 64 );
3238                 buffer_fadd(
3239                         _p,
3240                         "\"%s\".%s IS NULL",
3241                         class_info->alias,
3242                         osrfHashGet( field, "name" )
3243                 );
3244                 pred = buffer_release( _p );
3245         } else { // equality search
3246                 pred = searchSimplePredicate( "=", class_info->alias, field, node );
3247         }
3248
3249         return pred;
3250
3251 }
3252
3253
3254 /*
3255
3256 join : {
3257         acn : {
3258                 field : record,
3259                 fkey : id
3260                 type : left
3261                 filter_op : or
3262                 filter : { ... },
3263                 join : {
3264                         acp : {
3265                                 field : call_number,
3266                                 fkey : id,
3267                                 filter : { ... },
3268                         },
3269                 },
3270         },
3271         mrd : {
3272                 field : record,
3273                 type : inner
3274                 fkey : id,
3275                 filter : { ... },
3276         }
3277 }
3278
3279 */
3280
3281 static char* searchJOIN( const jsonObject* join_hash, const ClassInfo* left_info ) {
3282
3283         const jsonObject* working_hash;
3284         jsonObject* freeable_hash = NULL;
3285
3286         if( join_hash->type == JSON_HASH ) {
3287                 working_hash = join_hash;
3288         } else if( join_hash->type == JSON_STRING ) {
3289                 // turn it into a JSON_HASH by creating a wrapper
3290                 // around a copy of the original
3291                 const char* _tmp = jsonObjectGetString( join_hash );
3292                 freeable_hash = jsonNewObjectType( JSON_HASH );
3293                 jsonObjectSetKey( freeable_hash, _tmp, NULL );
3294                 working_hash = freeable_hash;
3295         } else {
3296                 osrfLogError(
3297                         OSRF_LOG_MARK,
3298                         "%s: JOIN failed; expected JSON object type not found",
3299                         modulename
3300                 );
3301                 return NULL;
3302         }
3303
3304         growing_buffer* join_buf = buffer_init( 128 );
3305         const char* leftclass = left_info->class_name;
3306
3307         jsonObject* snode = NULL;
3308         jsonIterator* search_itr = jsonNewIterator( working_hash );
3309
3310         while ( (snode = jsonIteratorNext( search_itr )) ) {
3311                 const char* right_alias = search_itr->key;
3312                 const char* class =
3313                                 jsonObjectGetString( jsonObjectGetKeyConst( snode, "class" ) );
3314                 if( ! class )
3315                         class = right_alias;
3316
3317                 const ClassInfo* right_info = add_joined_class( right_alias, class );
3318                 if( !right_info ) {
3319                         osrfLogError(
3320                                 OSRF_LOG_MARK,
3321                                 "%s: JOIN failed.  Class \"%s\" not resolved in IDL",
3322                                 modulename,
3323                                 search_itr->key
3324                         );
3325                         jsonIteratorFree( search_itr );
3326                         buffer_free( join_buf );
3327                         if( freeable_hash )
3328                                 jsonObjectFree( freeable_hash );
3329                         return NULL;
3330                 }
3331                 osrfHash* links    = right_info->links;
3332                 const char* table  = right_info->source_def;
3333
3334                 const char* fkey  = jsonObjectGetString( jsonObjectGetKeyConst( snode, "fkey" ) );
3335                 const char* field = jsonObjectGetString( jsonObjectGetKeyConst( snode, "field" ) );
3336
3337                 if( field && !fkey ) {
3338                         // Look up the corresponding join column in the IDL.
3339                         // The link must be defined in the child table,
3340                         // and point to the right parent table.
3341                         osrfHash* idl_link = (osrfHash*) osrfHashGet( links, field );
3342                         const char* reltype = NULL;
3343                         const char* other_class = NULL;
3344                         reltype = osrfHashGet( idl_link, "reltype" );
3345                         if( reltype && strcmp( reltype, "has_many" ) )
3346                                 other_class = osrfHashGet( idl_link, "class" );
3347                         if( other_class && !strcmp( other_class, leftclass ) )
3348                                 fkey = osrfHashGet( idl_link, "key" );
3349                         if( !fkey ) {
3350                                 osrfLogError(
3351                                         OSRF_LOG_MARK,
3352                                         "%s: JOIN failed.  No link defined from %s.%s to %s",
3353                                         modulename,
3354                                         class,
3355                                         field,
3356                                         leftclass
3357                                 );
3358                                 buffer_free( join_buf );
3359                                 if( freeable_hash )
3360                                         jsonObjectFree( freeable_hash );
3361                                 jsonIteratorFree( search_itr );
3362                                 return NULL;
3363                         }
3364
3365                 } else if( !field && fkey ) {
3366                         // Look up the corresponding join column in the IDL.
3367                         // The link must be defined in the child table,
3368                         // and point to the right parent table.
3369                         osrfHash* left_links = left_info->links;
3370                         osrfHash* idl_link = (osrfHash*) osrfHashGet( left_links, fkey );
3371                         const char* reltype = NULL;
3372                         const char* other_class = NULL;
3373                         reltype = osrfHashGet( idl_link, "reltype" );
3374                         if( reltype && strcmp( reltype, "has_many" ) )
3375                                 other_class = osrfHashGet( idl_link, "class" );
3376                         if( other_class && !strcmp( other_class, class ) )
3377                                 field = osrfHashGet( idl_link, "key" );
3378                         if( !field ) {
3379                                 osrfLogError(
3380                                         OSRF_LOG_MARK,
3381                                         "%s: JOIN failed.  No link defined from %s.%s to %s",
3382                                         modulename,
3383                                         leftclass,
3384                                         fkey,
3385                                         class
3386                                 );
3387                                 buffer_free( join_buf );
3388                                 if( freeable_hash )
3389                                         jsonObjectFree( freeable_hash );
3390                                 jsonIteratorFree( search_itr );
3391                                 return NULL;
3392                         }
3393
3394                 } else if( !field && !fkey ) {
3395                         osrfHash* left_links = left_info->links;
3396
3397                         // For each link defined for the left class:
3398                         // see if the link references the joined class
3399                         osrfHashIterator* itr = osrfNewHashIterator( left_links );
3400                         osrfHash* curr_link = NULL;
3401                         while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3402                                 const char* other_class = osrfHashGet( curr_link, "class" );
3403                                 if( other_class && !strcmp( other_class, class ) ) {
3404
3405                                         // In the IDL, the parent class doesn't always know then names of the child
3406                                         // columns that are pointing to it, so don't use that end of the link
3407                                         const char* reltype = osrfHashGet( curr_link, "reltype" );
3408                                         if( reltype && strcmp( reltype, "has_many" ) ) {
3409                                                 // Found a link between the classes
3410                                                 fkey = osrfHashIteratorKey( itr );
3411                                                 field = osrfHashGet( curr_link, "key" );
3412                                                 break;
3413                                         }
3414                                 }
3415                         }
3416                         osrfHashIteratorFree( itr );
3417
3418                         if( !field || !fkey ) {
3419                                 // Do another such search, with the classes reversed
3420
3421                                 // For each link defined for the joined class:
3422                                 // see if the link references the left class
3423                                 osrfHashIterator* itr = osrfNewHashIterator( links );
3424                                 osrfHash* curr_link = NULL;
3425                                 while( (curr_link = osrfHashIteratorNext( itr ) ) ) {
3426                                         const char* other_class = osrfHashGet( curr_link, "class" );
3427                                         if( other_class && !strcmp( other_class, leftclass ) ) {
3428
3429                                                 // In the IDL, the parent class doesn't know then names of the child
3430                                                 // columns that are pointing to it, so don't use that end of the link
3431                                                 const char* reltype = osrfHashGet( curr_link, "reltype" );
3432                                                 if( reltype && strcmp( reltype, "has_many" ) ) {
3433                                                         // Found a link between the classes
3434                                                         field = osrfHashIteratorKey( itr );
3435                                                         fkey = osrfHashGet( curr_link, "key" );
3436                                                         break;
3437                                                 }
3438                                         }
3439                                 }
3440                                 osrfHashIteratorFree( itr );
3441                         }
3442
3443                         if( !field || !fkey ) {
3444                                 osrfLogError(
3445                                         OSRF_LOG_MARK,
3446                                         "%s: JOIN failed.  No link defined between %s and %s",
3447                                         modulename,
3448                                         leftclass,
3449                                         class
3450                                 );
3451                                 buffer_free( join_buf );
3452                                 if( freeable_hash )
3453                                         jsonObjectFree( freeable_hash );
3454                                 jsonIteratorFree( search_itr );
3455                                 return NULL;
3456                         }
3457                 }
3458
3459                 const char* type = jsonObjectGetString( jsonObjectGetKeyConst( snode, "type" ) );
3460                 if( type ) {
3461                         if( !strcasecmp( type,"left" )) {
3462                                 buffer_add( join_buf, " LEFT JOIN" );
3463                         } else if( !strcasecmp( type,"right" )) {
3464                                 buffer_add( join_buf, " RIGHT JOIN" );
3465                         } else if( !strcasecmp( type,"full" )) {
3466                                 buffer_add( join_buf, " FULL JOIN" );
3467                         } else {
3468                                 buffer_add( join_buf, " INNER JOIN" );
3469                         }
3470                 } else {
3471                         buffer_add( join_buf, " INNER JOIN" );
3472                 }
3473
3474                 buffer_fadd( join_buf, " %s AS \"%s\" ON ( \"%s\".%s = \"%s\".%s",
3475                                         table, right_alias, right_alias, field, left_info->alias, fkey );
3476
3477                 // Add any other join conditions as specified by "filter"
3478                 const jsonObject* filter = jsonObjectGetKeyConst( snode, "filter" );
3479                 if( filter ) {
3480                         const char* filter_op = jsonObjectGetString(
3481                                 jsonObjectGetKeyConst( snode, "filter_op" ) );
3482                         if( filter_op && !strcasecmp( "or",filter_op )) {
3483                                 buffer_add( join_buf, " OR " );
3484                         } else {
3485                                 buffer_add( join_buf, " AND " );
3486                         }
3487
3488                         char* jpred = searchWHERE( filter, right_info, AND_OP_JOIN, NULL );
3489                         if( jpred ) {
3490                                 OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3491                                 OSRF_BUFFER_ADD( join_buf, jpred );
3492                                 free( jpred );
3493                         } else {
3494                                 osrfLogError(
3495                                         OSRF_LOG_MARK,
3496                                         "%s: JOIN failed.  Invalid conditional expression.",
3497                                         modulename
3498                                 );
3499                                 jsonIteratorFree( search_itr );
3500                                 buffer_free( join_buf );
3501                                 if( freeable_hash )
3502                                         jsonObjectFree( freeable_hash );
3503                                 return NULL;
3504                         }
3505                 }
3506
3507                 buffer_add( join_buf, " ) " );
3508
3509                 // Recursively add a nested join, if one is present
3510                 const jsonObject* join_filter = jsonObjectGetKeyConst( snode, "join" );
3511                 if( join_filter ) {
3512                         char* jpred = searchJOIN( join_filter, right_info );
3513                         if( jpred ) {
3514                                 OSRF_BUFFER_ADD_CHAR( join_buf, ' ' );
3515                                 OSRF_BUFFER_ADD( join_buf, jpred );
3516                                 free( jpred );
3517                         } else {
3518                                 osrfLogError( OSRF_LOG_MARK, "%s: Invalid nested join.", modulename );
3519                                 jsonIteratorFree( search_itr );
3520                                 buffer_free( join_buf );
3521                                 if( freeable_hash )
3522                                         jsonObjectFree( freeable_hash );
3523                                 return NULL;
3524                         }
3525                 }
3526         }
3527
3528         if( freeable_hash )
3529                 jsonObjectFree( freeable_hash );
3530         jsonIteratorFree( search_itr );
3531
3532         return buffer_release( join_buf );
3533 }
3534
3535 /*
3536
3537 { +class : { -or|-and : { field : { op : value }, ... } ... }, ... }
3538 { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }
3539 [ { +class : { -or|-and : [ { field : { op : value }, ... }, ...] ... }, ... }, ... ]
3540
3541 Generate code to express a set of conditions, as for a WHERE clause.  Parameters:
3542
3543 search_hash is the JSON expression of the conditions.
3544 meta is the class definition from the IDL, for the relevant table.
3545 opjoin_type indicates whether multiple conditions, if present, should be
3546         connected by AND or OR.
3547 osrfMethodContext is loaded with all sorts of stuff, but all we do with it here is
3548         to pass it to other functions -- and all they do with it is to use the session
3549         and request members to send error messages back to the client.
3550
3551 */
3552
3553 static char* searchWHERE( const jsonObject* search_hash, const ClassInfo* class_info,
3554                 int opjoin_type, osrfMethodContext* ctx ) {
3555
3556         osrfLogDebug(
3557                 OSRF_LOG_MARK,
3558                 "%s: Entering searchWHERE; search_hash addr = %p, meta addr = %p, "
3559                 "opjoin_type = %d, ctx addr = %p",
3560                 modulename,
3561                 search_hash,
3562                 class_info->class_def,
3563                 opjoin_type,
3564                 ctx
3565         );
3566
3567         growing_buffer* sql_buf = buffer_init( 128 );
3568
3569         jsonObject* node = NULL;
3570
3571         int first = 1;
3572         if( search_hash->type == JSON_ARRAY ) {
3573                 if( 0 == search_hash->size ) {
3574                         osrfLogError(
3575                                 OSRF_LOG_MARK,
3576                                 "%s: Invalid predicate structure: empty JSON array",
3577                                 modulename
3578                         );
3579                         buffer_free( sql_buf );
3580                         return NULL;
3581                 }
3582
3583                 unsigned long i = 0;
3584                 while(( node = jsonObjectGetIndex( search_hash, i++ ) )) {
3585                         if( first ) {
3586                                 first = 0;
3587                         } else {
3588                                 if( opjoin_type == OR_OP_JOIN )
3589                                         buffer_add( sql_buf, " OR " );
3590                                 else
3591                                         buffer_add( sql_buf, " AND " );
3592                         }
3593
3594                         char* subpred = searchWHERE( node, class_info, opjoin_type, ctx );
3595                         if( ! subpred ) {
3596                                 buffer_free( sql_buf );
3597                                 return NULL;
3598                         }
3599
3600                         buffer_fadd( sql_buf, "( %s )", subpred );
3601                         free( subpred );
3602                 }
3603
3604         } else if( search_hash->type == JSON_HASH ) {
3605                 osrfLogDebug( OSRF_LOG_MARK,
3606                         "%s: In WHERE clause, condition type is JSON_HASH", modulename );
3607                 jsonIterator* search_itr = jsonNewIterator( search_hash );
3608                 if( !jsonIteratorHasNext( search_itr ) ) {
3609                         osrfLogError(
3610                                 OSRF_LOG_MARK,
3611                                 "%s: Invalid predicate structure: empty JSON object",
3612                                 modulename
3613                         );
3614                         jsonIteratorFree( search_itr );
3615                         buffer_free( sql_buf );
3616                         return NULL;
3617                 }
3618
3619                 while( (node = jsonIteratorNext( search_itr )) ) {
3620
3621                         if( first ) {
3622                                 first = 0;
3623                         } else {
3624                                 if( opjoin_type == OR_OP_JOIN )
3625                                         buffer_add( sql_buf, " OR " );
3626                                 else
3627                                         buffer_add( sql_buf, " AND " );
3628                         }
3629
3630                         if( '+' == search_itr->key[ 0 ] ) {
3631
3632                                 // This plus sign prefixes a class name or other table alias;
3633                                 // make sure the table alias is in scope
3634                                 ClassInfo* alias_info = search_all_alias( search_itr->key + 1 );
3635                                 if( ! alias_info ) {
3636                                         osrfLogError(
3637                                                          OSRF_LOG_MARK,
3638                                                         "%s: Invalid table alias \"%s\" in WHERE clause",
3639                                                         modulename,
3640                                                         search_itr->key + 1
3641                                         );
3642                                         jsonIteratorFree( search_itr );
3643                                         buffer_free( sql_buf );
3644                                         return NULL;
3645                                 }
3646
3647                                 if( node->type == JSON_STRING ) {
3648                                         // It's the name of a column; make sure it belongs to the class
3649                                         const char* fieldname = jsonObjectGetString( node );
3650                                         if( ! osrfHashGet( alias_info->fields, fieldname ) ) {
3651                                                 osrfLogError(
3652                                                         OSRF_LOG_MARK,
3653                                                         "%s: Invalid column name \"%s\" in WHERE clause "
3654                                                         "for table alias \"%s\"",
3655                                                         modulename,
3656                                                         fieldname,
3657                                                         alias_info->alias
3658                                                 );
3659                                                 jsonIteratorFree( search_itr );
3660                                                 buffer_free( sql_buf );
3661                                                 return NULL;
3662                                         }
3663
3664                                         buffer_fadd( sql_buf, " \"%s\".%s ", alias_info->alias, fieldname );
3665                                 } else {
3666                                         // It's something more complicated
3667                                         char* subpred = searchWHERE( node, alias_info, AND_OP_JOIN, ctx );
3668                                         if( ! subpred ) {
3669                                                 jsonIteratorFree( search_itr );
3670                                                 buffer_free( sql_buf );
3671                                                 return NULL;
3672                                         }
3673
3674                                         buffer_fadd( sql_buf, "( %s )", subpred );
3675                                         free( subpred );
3676                                 }
3677                         } else if( '-' == search_itr->key[ 0 ] ) {
3678                                 if( !strcasecmp( "-or", search_itr->key )) {
3679                                         char* subpred = searchWHERE( node, class_info, OR_OP_JOIN, ctx );
3680                                         if( ! subpred ) {
3681                                                 jsonIteratorFree( search_itr );
3682                                                 buffer_free( sql_buf );
3683                                                 return NULL;
3684                                         }
3685
3686                                         buffer_fadd( sql_buf, "( %s )", subpred );
3687                                         free( subpred );
3688                                 } else if( !strcasecmp( "-and", search_itr->key )) {
3689                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3690                                         if( ! subpred ) {
3691                                                 jsonIteratorFree( search_itr );
3692                                                 buffer_free( sql_buf );
3693                                                 return NULL;
3694                                         }
3695
3696                                         buffer_fadd( sql_buf, "( %s )", subpred );
3697                                         free( subpred );
3698                                 } else if( !strcasecmp("-not",search_itr->key) ) {
3699                                         char* subpred = searchWHERE( node, class_info, AND_OP_JOIN, ctx );
3700                                         if( ! subpred ) {
3701                                                 jsonIteratorFree( search_itr );
3702                                                 buffer_free( sql_buf );
3703                                                 return NULL;
3704                                         }
3705
3706                                         buffer_fadd( sql_buf, " NOT ( %s )", subpred );
3707                                         free( subpred );
3708                                 } else if( !strcasecmp( "-exists", search_itr->key )) {
3709                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3710                                         if( ! subpred ) {
3711                                                 jsonIteratorFree( search_itr );
3712                                                 buffer_free( sql_buf );
3713                                                 return NULL;
3714                                         }
3715
3716                                         buffer_fadd( sql_buf, "EXISTS ( %s )", subpred );
3717                                         free( subpred );
3718                                 } else if( !strcasecmp("-not-exists", search_itr->key )) {
3719                                         char* subpred = buildQuery( ctx, node, SUBSELECT );
3720                                         if( ! subpred ) {
3721                                                 jsonIteratorFree( search_itr );
3722                                                 buffer_free( sql_buf );
3723                                                 return NULL;
3724                                         }
3725
3726                                         buffer_fadd( sql_buf, "NOT EXISTS ( %s )", subpred );
3727                                         free( subpred );
3728                                 } else {     // Invalid "minus" operator
3729                                         osrfLogError(
3730                                                          OSRF_LOG_MARK,
3731                                                         "%s: Invalid operator \"%s\" in WHERE clause",
3732                                                         modulename,
3733                                                         search_itr->key
3734                                         );
3735                                         jsonIteratorFree( search_itr );
3736                                         buffer_free( sql_buf );
3737                                         return NULL;
3738                                 }
3739
3740                         } else {
3741
3742                                 const char* class = class_info->class_name;
3743                                 osrfHash* fields = class_info->fields;
3744                                 osrfHash* field = osrfHashGet( fields, search_itr->key );
3745
3746                                 if( !field ) {
3747                                         const char* table = class_info->source_def;
3748                                         osrfLogError(
3749                                                 OSRF_LOG_MARK,
3750                                                 "%s: Attempt to reference non-existent column \"%s\" on %s (%s)",
3751                                                 modulename,
3752                                                 search_itr->key,
3753                                                 table ? table : "?",
3754                                                 class ? class : "?"
3755                                         );
3756                                         jsonIteratorFree( search_itr );
3757                                         buffer_free( sql_buf );
3758                                         return NULL;
3759                                 }
3760
3761                                 char* subpred = searchPredicate( class_info, field, node, ctx );
3762                                 if( ! subpred ) {
3763                                         buffer_free( sql_buf );
3764                                         jsonIteratorFree( search_itr );
3765                                         return NULL;
3766                                 }
3767
3768                                 buffer_add( sql_buf, subpred );
3769                                 free( subpred );
3770                         }
3771                 }
3772                 jsonIteratorFree( search_itr );
3773
3774         } else {
3775                 // ERROR ... only hash and array allowed at this level
3776                 char* predicate_string = jsonObjectToJSON( search_hash );
3777                 osrfLogError(
3778                         OSRF_LOG_MARK,
3779                         "%s: Invalid predicate structure: %s",
3780                         modulename,
3781                         predicate_string
3782                 );
3783                 buffer_free( sql_buf );
3784                 free( predicate_string );
3785                 return NULL;
3786         }
3787
3788         return buffer_release( sql_buf );
3789 }
3790
3791 /* Build a JSON_ARRAY of field names for a given table alias
3792 */
3793 static jsonObject* defaultSelectList( const char* table_alias ) {
3794
3795         if( ! table_alias )
3796                 table_alias = "";
3797
3798         ClassInfo* class_info = search_all_alias( table_alias );
3799         if( ! class_info ) {
3800                 osrfLogError(
3801                         OSRF_LOG_MARK,
3802                         "%s: Can't build default SELECT clause for \"%s\"; no such table alias",
3803                         modulename,
3804                         table_alias
3805                 );
3806                 return NULL;
3807         }
3808
3809         jsonObject* array = jsonNewObjectType( JSON_ARRAY );
3810         osrfHash* field_def = NULL;
3811         osrfHashIterator* field_itr = osrfNewHashIterator( class_info->fields );
3812         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
3813                 const char* field_name = osrfHashIteratorKey( field_itr );
3814                 if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
3815                         jsonObjectPush( array, jsonNewObject( field_name ) );
3816                 }
3817         }
3818         osrfHashIteratorFree( field_itr );
3819
3820         return array;
3821 }
3822
3823 // Translate a jsonObject into a UNION, INTERSECT, or EXCEPT query.
3824 // The jsonObject must be a JSON_HASH with an single entry for "union",
3825 // "intersect", or "except".  The data associated with this key must be an
3826 // array of hashes, each hash being a query.
3827 // Also allowed but currently ignored: entries for "order_by" and "alias".
3828 static char* doCombo( osrfMethodContext* ctx, jsonObject* combo, int flags ) {
3829         // Sanity check
3830         if( ! combo || combo->type != JSON_HASH )
3831                 return NULL;      // should be impossible; validated by caller
3832
3833         const jsonObject* query_array = NULL;   // array of subordinate queries
3834         const char* op = NULL;     // name of operator, e.g. UNION
3835         const char* alias = NULL;  // alias for the query (needed for ORDER BY)
3836         int op_count = 0;          // for detecting conflicting operators
3837         int excepting = 0;         // boolean
3838         int all = 0;               // boolean
3839         jsonObject* order_obj = NULL;
3840
3841         // Identify the elements in the hash
3842         jsonIterator* query_itr = jsonNewIterator( combo );
3843         jsonObject* curr_obj = NULL;
3844         while( (curr_obj = jsonIteratorNext( query_itr ) ) ) {
3845                 if( ! strcmp( "union", query_itr->key ) ) {
3846                         ++op_count;
3847                         op = " UNION ";
3848                         query_array = curr_obj;
3849                 } else if( ! strcmp( "intersect", query_itr->key ) ) {
3850                         ++op_count;
3851                         op = " INTERSECT ";
3852                         query_array = curr_obj;
3853                 } else if( ! strcmp( "except", query_itr->key ) ) {
3854                         ++op_count;
3855                         op = " EXCEPT ";
3856                         excepting = 1;
3857                         query_array = curr_obj;
3858                 } else if( ! strcmp( "order_by", query_itr->key ) ) {
3859                         osrfLogWarning(
3860                                 OSRF_LOG_MARK,
3861                                 "%s: ORDER BY not supported for UNION, INTERSECT, or EXCEPT",
3862                                 modulename
3863                         );
3864                         order_obj = curr_obj;
3865                 } else if( ! strcmp( "alias", query_itr->key ) ) {
3866                         if( curr_obj->type != JSON_STRING ) {
3867                                 jsonIteratorFree( query_itr );
3868                                 return NULL;
3869                         }
3870                         alias = jsonObjectGetString( curr_obj );
3871                 } else if( ! strcmp( "all", query_itr->key ) ) {
3872                         if( obj_is_true( curr_obj ) )
3873                                 all = 1;
3874                 } else {
3875                         if( ctx )
3876                                 osrfAppSessionStatus(
3877                                         ctx->session,
3878                                         OSRF_STATUS_INTERNALSERVERERROR,
3879                                         "osrfMethodException",
3880                                         ctx->request,
3881                                         "Malformed query; unexpected entry in query object"
3882                                 );
3883                         osrfLogError(
3884                                 OSRF_LOG_MARK,
3885                                 "%s: Unexpected entry for \"%s\" in%squery",
3886                                 modulename,
3887                                 query_itr->key,
3888                                 op
3889                         );
3890                         jsonIteratorFree( query_itr );
3891                         return NULL;
3892                 }
3893         }
3894         jsonIteratorFree( query_itr );
3895
3896         // More sanity checks
3897         if( ! query_array ) {
3898                 if( ctx )
3899                         osrfAppSessionStatus(
3900                                 ctx->session,
3901                                 OSRF_STATUS_INTERNALSERVERERROR,
3902                                 "osrfMethodException",
3903                                 ctx->request,
3904                                 "Expected UNION, INTERSECT, or EXCEPT operator not found"
3905                         );
3906                 osrfLogError(
3907                         OSRF_LOG_MARK,
3908                         "%s: Expected UNION, INTERSECT, or EXCEPT operator not found",
3909                         modulename
3910                 );
3911                 return NULL;        // should be impossible...
3912         } else if( op_count > 1 ) {
3913                 if( ctx )
3914                                 osrfAppSessionStatus(
3915                                 ctx->session,
3916                                 OSRF_STATUS_INTERNALSERVERERROR,
3917                                 "osrfMethodException",
3918                                 ctx->request,
3919                                 "Found more than one of UNION, INTERSECT, and EXCEPT in same query"
3920                         );
3921                 osrfLogError(
3922                         OSRF_LOG_MARK,
3923                         "%s: Found more than one of UNION, INTERSECT, and EXCEPT in same query",
3924                         modulename
3925                 );
3926                 return NULL;
3927         } if( query_array->type != JSON_ARRAY ) {
3928                 if( ctx )
3929                                 osrfAppSessionStatus(
3930                                 ctx->session,
3931                                 OSRF_STATUS_INTERNALSERVERERROR,
3932                                 "osrfMethodException",
3933                                 ctx->request,
3934                                 "Malformed query: expected array of queries under UNION, INTERSECT or EXCEPT"
3935                         );
3936                 osrfLogError(
3937                         OSRF_LOG_MARK,
3938                         "%s: Expected JSON_ARRAY of queries for%soperator; found %s",
3939                         modulename,
3940                         op,
3941                         json_type( query_array->type )
3942                 );
3943                 return NULL;
3944         } if( query_array->size < 2 ) {
3945                 if( ctx )
3946                         osrfAppSessionStatus(
3947                                 ctx->session,
3948                                 OSRF_STATUS_INTERNALSERVERERROR,
3949                                 "osrfMethodException",
3950                                 ctx->request,
3951                                 "UNION, INTERSECT or EXCEPT requires multiple queries as operands"
3952                         );
3953                 osrfLogError(
3954                         OSRF_LOG_MARK,
3955                         "%s:%srequires multiple queries as operands",
3956                         modulename,
3957                         op
3958                 );
3959                 return NULL;
3960         } else if( excepting && query_array->size > 2 ) {
3961                 if( ctx )
3962                         osrfAppSessionStatus(
3963                                 ctx->session,
3964                                 OSRF_STATUS_INTERNALSERVERERROR,
3965                                 "osrfMethodException",
3966                                 ctx->request,
3967                                 "EXCEPT operator has too many queries as operands"
3968                         );
3969                 osrfLogError(
3970                         OSRF_LOG_MARK,
3971                         "%s:EXCEPT operator has too many queries as operands",
3972                         modulename
3973                 );
3974                 return NULL;
3975         } else if( order_obj && ! alias ) {
3976                 if( ctx )
3977                         osrfAppSessionStatus(
3978                                 ctx->session,
3979                                 OSRF_STATUS_INTERNALSERVERERROR,
3980                                 "osrfMethodException",
3981                                 ctx->request,
3982                                 "ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT"
3983                         );
3984                 osrfLogError(
3985                         OSRF_LOG_MARK,
3986                         "%s:ORDER BY requires an alias for a UNION, INTERSECT, or EXCEPT",
3987                         modulename
3988                 );
3989                 return NULL;
3990         }
3991
3992         // So far so good.  Now build the SQL.
3993         growing_buffer* sql = buffer_init( 256 );
3994
3995         // If we nested inside another UNION, INTERSECT, or EXCEPT,
3996         // Add a layer of parentheses
3997         if( flags & SUBCOMBO )
3998                 OSRF_BUFFER_ADD( sql, "( " );
3999
4000         // Traverse the query array.  Each entry should be a hash.
4001         int first = 1;   // boolean
4002         int i = 0;
4003         jsonObject* query = NULL;
4004         while( (query = jsonObjectGetIndex( query_array, i++ )) ) {
4005                 if( query->type != JSON_HASH ) {
4006                         if( ctx )
4007                                 osrfAppSessionStatus(
4008                                         ctx->session,
4009                                         OSRF_STATUS_INTERNALSERVERERROR,
4010                                         "osrfMethodException",
4011                                         ctx->request,
4012                                         "Malformed query under UNION, INTERSECT or EXCEPT"
4013                                 );
4014                         osrfLogError(
4015                                 OSRF_LOG_MARK,
4016                                 "%s: Malformed query under%s -- expected JSON_HASH, found %s",
4017                                 modulename,
4018                                 op,
4019                                 json_type( query->type )
4020                         );
4021                         buffer_free( sql );
4022                         return NULL;
4023                 }
4024
4025                 if( first )
4026                         first = 0;
4027                 else {
4028                         OSRF_BUFFER_ADD( sql, op );
4029                         if( all )
4030                                 OSRF_BUFFER_ADD( sql, "ALL " );
4031                 }
4032
4033                 char* query_str = buildQuery( ctx, query, SUBSELECT | SUBCOMBO );
4034                 if( ! query_str ) {
4035                         osrfLogError(
4036                                 OSRF_LOG_MARK,
4037                                 "%s: Error building query under%s",
4038                                 modulename,
4039                                 op
4040                         );
4041                         buffer_free( sql );
4042                         return NULL;
4043                 }
4044
4045                 OSRF_BUFFER_ADD( sql, query_str );
4046         }
4047
4048         if( flags & SUBCOMBO )
4049                 OSRF_BUFFER_ADD_CHAR( sql, ')' );
4050
4051         if( !(flags & SUBSELECT) )
4052                 OSRF_BUFFER_ADD_CHAR( sql, ';' );
4053
4054         return buffer_release( sql );
4055 }
4056
4057 // Translate a jsonObject into a SELECT, UNION, INTERSECT, or EXCEPT query.
4058 // The jsonObject must be a JSON_HASH with an entry for "from", "union", "intersect",
4059 // or "except" to indicate the type of query.
4060 char* buildQuery( osrfMethodContext* ctx, jsonObject* query, int flags ) {
4061         // Sanity checks
4062         if( ! query ) {
4063                 if( ctx )
4064                         osrfAppSessionStatus(
4065                                 ctx->session,
4066                                 OSRF_STATUS_INTERNALSERVERERROR,
4067                                 "osrfMethodException",
4068                                 ctx->request,
4069                                 "Malformed query; no query object"
4070                         );
4071                 osrfLogError( OSRF_LOG_MARK, "%s: Null pointer to query object", modulename );
4072                 return NULL;
4073         } else if( query->type != JSON_HASH ) {
4074                 if( ctx )
4075                         osrfAppSessionStatus(
4076                                 ctx->session,
4077                                 OSRF_STATUS_INTERNALSERVERERROR,
4078                                 "osrfMethodException",
4079                                 ctx->request,
4080                                 "Malformed query object"
4081                         );
4082                 osrfLogError(
4083                         OSRF_LOG_MARK,
4084                         "%s: Query object is %s instead of JSON_HASH",
4085                         modulename,
4086                         json_type( query->type )
4087                 );
4088                 return NULL;
4089         }
4090
4091         // Determine what kind of query it purports to be, and dispatch accordingly.
4092         if( jsonObjectGetKeyConst( query, "union" ) ||
4093                 jsonObjectGetKeyConst( query, "intersect" ) ||
4094                 jsonObjectGetKeyConst( query, "except" )) {
4095                 return doCombo( ctx, query, flags );
4096         } else {
4097                 // It is presumably a SELECT query
4098
4099                 // Push a node onto the stack for the current query.  Every level of
4100                 // subquery gets its own QueryFrame on the Stack.
4101                 push_query_frame();
4102
4103                 // Build an SQL SELECT statement
4104                 char* sql = SELECT(
4105                         ctx,
4106                         jsonObjectGetKey( query, "select" ),
4107                         jsonObjectGetKeyConst( query, "from" ),
4108                         jsonObjectGetKeyConst( query, "where" ),
4109                         jsonObjectGetKeyConst( query, "having" ),
4110                         jsonObjectGetKeyConst( query, "order_by" ),
4111                         jsonObjectGetKeyConst( query, "limit" ),
4112                         jsonObjectGetKeyConst( query, "offset" ),
4113                         flags
4114                 );
4115                 pop_query_frame();
4116                 return sql;
4117         }
4118 }
4119
4120 char* SELECT (
4121                 /* method context */ osrfMethodContext* ctx,
4122
4123                 /* SELECT   */ jsonObject* selhash,
4124                 /* FROM     */ const jsonObject* join_hash,
4125                 /* WHERE    */ const jsonObject* search_hash,
4126                 /* HAVING   */ const jsonObject* having_hash,
4127                 /* ORDER BY */ const jsonObject* order_hash,
4128                 /* LIMIT    */ const jsonObject* limit,
4129                 /* OFFSET   */ const jsonObject* offset,
4130                 /* flags    */ int flags
4131 ) {
4132         const char* locale = osrf_message_get_last_locale();
4133
4134         // general tmp objects
4135         const jsonObject* tmp_const;
4136         jsonObject* selclass = NULL;
4137         jsonObject* snode = NULL;
4138         jsonObject* onode = NULL;
4139
4140         char* string = NULL;
4141         int from_function = 0;
4142         int first = 1;
4143         int gfirst = 1;
4144         //int hfirst = 1;
4145
4146         osrfLogDebug(OSRF_LOG_MARK, "cstore SELECT locale: %s", locale ? locale : "(none)" );
4147
4148         // punt if there's no FROM clause
4149         if( !join_hash || ( join_hash->type == JSON_HASH && !join_hash->size )) {
4150                 osrfLogError(
4151                         OSRF_LOG_MARK,
4152                         "%s: FROM clause is missing or empty",
4153                         modulename
4154                 );
4155                 if( ctx )
4156                         osrfAppSessionStatus(
4157                                 ctx->session,
4158                                 OSRF_STATUS_INTERNALSERVERERROR,
4159                                 "osrfMethodException",
4160                                 ctx->request,
4161                                 "FROM clause is missing or empty in JSON query"
4162                         );
4163                 return NULL;
4164         }
4165
4166         // the core search class
4167         const char* core_class = NULL;
4168
4169         // get the core class -- the only key of the top level FROM clause, or a string
4170         if( join_hash->type == JSON_HASH ) {
4171                 jsonIterator* tmp_itr = jsonNewIterator( join_hash );
4172                 snode = jsonIteratorNext( tmp_itr );
4173
4174                 // Populate the current QueryFrame with information
4175                 // about the core class
4176                 if( add_query_core( NULL, tmp_itr->key ) ) {
4177                         if( ctx )
4178                                 osrfAppSessionStatus(
4179                                         ctx->session,
4180                                         OSRF_STATUS_INTERNALSERVERERROR,
4181                                         "osrfMethodException",
4182                                         ctx->request,
4183                                         "Unable to look up core class"
4184                                 );
4185                         return NULL;
4186                 }
4187                 core_class = curr_query->core.class_name;
4188                 join_hash = snode;
4189
4190                 jsonObject* extra = jsonIteratorNext( tmp_itr );
4191
4192                 jsonIteratorFree( tmp_itr );
4193                 snode = NULL;
4194
4195                 // There shouldn't be more than one entry in join_hash
4196                 if( extra ) {
4197                         osrfLogError(
4198                                 OSRF_LOG_MARK,
4199                                 "%s: Malformed FROM clause: extra entry in JSON_HASH",
4200                                 modulename
4201                         );
4202                         if( ctx )
4203                                 osrfAppSessionStatus(
4204                                         ctx->session,
4205                                         OSRF_STATUS_INTERNALSERVERERROR,
4206                                         "osrfMethodException",
4207                                         ctx->request,
4208                                         "Malformed FROM clause in JSON query"
4209                                 );
4210                         return NULL;    // Malformed join_hash; extra entry
4211                 }
4212         } else if( join_hash->type == JSON_ARRAY ) {
4213                 // We're selecting from a function, not from a table
4214                 from_function = 1;
4215                 core_class = jsonObjectGetString( jsonObjectGetIndex( join_hash, 0 ));
4216                 selhash = NULL;
4217
4218         } else if( join_hash->type == JSON_STRING ) {
4219                 // Populate the current QueryFrame with information
4220                 // about the core class
4221                 core_class = jsonObjectGetString( join_hash );
4222                 join_hash = NULL;
4223                 if( add_query_core( NULL, core_class ) ) {
4224                         if( ctx )
4225                                 osrfAppSessionStatus(
4226                                         ctx->session,
4227                                         OSRF_STATUS_INTERNALSERVERERROR,
4228                                         "osrfMethodException",
4229                                         ctx->request,
4230                                         "Unable to look up core class"
4231                                 );
4232                         return NULL;
4233                 }
4234         }
4235         else {
4236                 osrfLogError(
4237                         OSRF_LOG_MARK,
4238                         "%s: FROM clause is unexpected JSON type: %s",
4239                         modulename,
4240                         json_type( join_hash->type )
4241                 );
4242                 if( ctx )
4243                         osrfAppSessionStatus(
4244                                 ctx->session,
4245                                 OSRF_STATUS_INTERNALSERVERERROR,
4246                                 "osrfMethodException",
4247                                 ctx->request,
4248                                 "Ill-formed FROM clause in JSON query"
4249                         );
4250                 return NULL;
4251         }
4252
4253         // Build the join clause, if any, while filling out the list
4254         // of joined classes in the current QueryFrame.
4255         char* join_clause = NULL;
4256         if( join_hash && ! from_function ) {
4257
4258                 join_clause = searchJOIN( join_hash, &curr_query->core );
4259                 if( ! join_clause ) {
4260                         if( ctx )
4261                                 osrfAppSessionStatus(
4262                                         ctx->session,
4263                                         OSRF_STATUS_INTERNALSERVERERROR,
4264                                         "osrfMethodException",
4265                                         ctx->request,
4266                                         "Unable to construct JOIN clause(s)"
4267                                 );
4268                         return NULL;
4269                 }
4270         }
4271
4272         // For in case we don't get a select list
4273         jsonObject* defaultselhash = NULL;
4274
4275         // if there is no select list, build a default select list ...
4276         if( !selhash && !from_function ) {
4277                 jsonObject* default_list = defaultSelectList( core_class );
4278                 if( ! default_list ) {
4279                         if( ctx ) {
4280                                 osrfAppSessionStatus(
4281                                         ctx->session,
4282                                         OSRF_STATUS_INTERNALSERVERERROR,
4283                                         "osrfMethodException",
4284                                         ctx->request,
4285                                         "Unable to build default SELECT clause in JSON query"
4286                                 );
4287                                 free( join_clause );
4288                                 return NULL;
4289                         }
4290                 }
4291
4292                 selhash = defaultselhash = jsonNewObjectType( JSON_HASH );
4293                 jsonObjectSetKey( selhash, core_class, default_list );
4294         }
4295
4296         // The SELECT clause can be encoded only by a hash
4297         if( !from_function && selhash->type != JSON_HASH ) {
4298                 osrfLogError(
4299                         OSRF_LOG_MARK,
4300                         "%s: Expected JSON_HASH for SELECT clause; found %s",
4301                         modulename,
4302                         json_type( selhash->type )
4303                 );
4304
4305                 if( ctx )
4306                         osrfAppSessionStatus(
4307                                 ctx->session,
4308                                 OSRF_STATUS_INTERNALSERVERERROR,
4309                                 "osrfMethodException",
4310                                 ctx->request,
4311                                 "Malformed SELECT clause in JSON query"
4312                         );
4313                 free( join_clause );
4314                 return NULL;
4315         }
4316
4317         // If you see a null or wild card specifier for the core class, or an
4318         // empty array, replace it with a default SELECT list
4319         tmp_const = jsonObjectGetKeyConst( selhash, core_class );
4320         if( tmp_const ) {
4321                 int default_needed = 0;   // boolean
4322                 if( JSON_STRING == tmp_const->type
4323                         && !strcmp( "*", jsonObjectGetString( tmp_const ) ))
4324                                 default_needed = 1;
4325                 else if( JSON_NULL == tmp_const->type )
4326                         default_needed = 1;
4327
4328                 if( default_needed ) {
4329                         // Build a default SELECT list
4330                         jsonObject* default_list = defaultSelectList( core_class );
4331                         if( ! default_list ) {
4332                                 if( ctx ) {
4333                                         osrfAppSessionStatus(
4334                                                 ctx->session,
4335                                                 OSRF_STATUS_INTERNALSERVERERROR,
4336                                                 "osrfMethodException",
4337                                                 ctx->request,
4338                                                 "Can't build default SELECT clause in JSON query"
4339                                         );
4340                                         free( join_clause );
4341                                         return NULL;
4342                                 }
4343                         }
4344
4345                         jsonObjectSetKey( selhash, core_class, default_list );
4346                 }
4347         }
4348
4349         // temp buffers for the SELECT list and GROUP BY clause
4350         growing_buffer* select_buf = buffer_init( 128 );
4351         growing_buffer* group_buf  = buffer_init( 128 );
4352
4353         int aggregate_found = 0;     // boolean
4354
4355         // Build a select list
4356         if( from_function )   // From a function we select everything
4357                 OSRF_BUFFER_ADD_CHAR( select_buf, '*' );
4358         else {
4359
4360                 // Build the SELECT list as SQL
4361             int sel_pos = 1;
4362             first = 1;
4363             gfirst = 1;
4364             jsonIterator* selclass_itr = jsonNewIterator( selhash );
4365             while ( (selclass = jsonIteratorNext( selclass_itr )) ) {    // For each class
4366
4367                         const char* cname = selclass_itr->key;
4368
4369                         // Make sure the target relation is in the FROM clause.
4370
4371                         // At this point join_hash is a step down from the join_hash we
4372                         // received as a parameter.  If the original was a JSON_STRING,
4373                         // then json_hash is now NULL.  If the original was a JSON_HASH,
4374                         // then json_hash is now the first (and only) entry in it,
4375                         // denoting the core class.  We've already excluded the
4376                         // possibility that the original was a JSON_ARRAY, because in
4377                         // that case from_function would be non-NULL, and we wouldn't
4378                         // be here.
4379
4380                         // If the current table alias isn't in scope, bail out
4381                         ClassInfo* class_info = search_alias( cname );
4382                         if( ! class_info ) {
4383                                 osrfLogError(
4384                                         OSRF_LOG_MARK,
4385                                         "%s: SELECT clause references class not in FROM clause: \"%s\"",
4386                                         modulename,
4387                                         cname
4388                                 );
4389                                 if( ctx )
4390                                         osrfAppSessionStatus(
4391                                                 ctx->session,
4392                                                 OSRF_STATUS_INTERNALSERVERERROR,
4393                                                 "osrfMethodException",
4394                                                 ctx->request,
4395                                                 "Selected class not in FROM clause in JSON query"
4396                                         );
4397                                 jsonIteratorFree( selclass_itr );
4398                                 buffer_free( select_buf );
4399                                 buffer_free( group_buf );
4400                                 if( defaultselhash )
4401                                         jsonObjectFree( defaultselhash );
4402                                 free( join_clause );
4403                                 return NULL;
4404                         }
4405
4406                         if( selclass->type != JSON_ARRAY ) {
4407                                 osrfLogError(
4408                                         OSRF_LOG_MARK,
4409                                         "%s: Malformed SELECT list for class \"%s\"; not an array",
4410                                         modulename,
4411                                         cname
4412                                 );
4413                                 if( ctx )
4414                                         osrfAppSessionStatus(
4415                                                 ctx->session,
4416                                                 OSRF_STATUS_INTERNALSERVERERROR,
4417                                                 "osrfMethodException",
4418                                                 ctx->request,
4419                                                 "Selected class not in FROM clause in JSON query"
4420                                         );
4421
4422                                 jsonIteratorFree( selclass_itr );
4423                                 buffer_free( select_buf );
4424                                 buffer_free( group_buf );
4425                                 if( defaultselhash )
4426                                         jsonObjectFree( defaultselhash );
4427                                 free( join_clause );
4428                                 return NULL;
4429                         }
4430
4431                         // Look up some attributes of the current class
4432                         osrfHash* idlClass        = class_info->class_def;
4433                         osrfHash* class_field_set = class_info->fields;
4434                         const char* class_pkey    = osrfHashGet( idlClass, "primarykey" );
4435                         const char* class_tname   = osrfHashGet( idlClass, "tablename" );
4436
4437                         if( 0 == selclass->size ) {
4438                                 osrfLogWarning(
4439                                         OSRF_LOG_MARK,
4440                                         "%s: No columns selected from \"%s\"",
4441                                         modulename,
4442                                         cname
4443                                 );
4444                         }
4445
4446                         // stitch together the column list for the current table alias...
4447                         unsigned long field_idx = 0;
4448                         jsonObject* selfield = NULL;
4449                         while(( selfield = jsonObjectGetIndex( selclass, field_idx++ ) )) {
4450
4451                                 // If we need a separator comma, add one
4452                                 if( first ) {
4453                                         first = 0;
4454                                 } else {
4455                                         OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
4456                                 }
4457
4458                                 // if the field specification is a string, add it to the list
4459                                 if( selfield->type == JSON_STRING ) {
4460
4461                                         // Look up the field in the IDL
4462                                         const char* col_name = jsonObjectGetString( selfield );
4463                                         osrfHash* field_def = NULL;
4464
4465                                         if (!osrfStringArrayContains(
4466                                                         osrfHashGet(
4467                                                                 osrfHashGet( class_field_set, col_name ),
4468                                                                 "suppress_controller"),
4469                                                         modulename
4470                                         ))
4471                                                 field_def = osrfHashGet( class_field_set, col_name );
4472
4473                                         if( !field_def ) {
4474                                                 // No such field in current class
4475                                                 osrfLogError(
4476                                                         OSRF_LOG_MARK,
4477                                                         "%s: Selected column \"%s\" not defined in IDL for class \"%s\"",
4478                                                         modulename,
4479                                                         col_name,
4480                                                         cname
4481                                                 );
4482                                                 if( ctx )
4483                                                         osrfAppSessionStatus(
4484                                                                 ctx->session,
4485                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4486                                                                 "osrfMethodException",
4487                                                                 ctx->request,
4488                                                                 "Selected column not defined in JSON query"
4489                                                         );
4490                                                 jsonIteratorFree( selclass_itr );
4491                                                 buffer_free( select_buf );
4492                                                 buffer_free( group_buf );
4493                                                 if( defaultselhash )
4494                                                         jsonObjectFree( defaultselhash );
4495                                                 free( join_clause );
4496                                                 return NULL;
4497                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
4498                                                 // Virtual field not allowed
4499                                                 osrfLogError(
4500                                                         OSRF_LOG_MARK,
4501                                                         "%s: Selected column \"%s\" for class \"%s\" is virtual",
4502                                                         modulename,
4503                                                         col_name,
4504                                                         cname
4505                                                 );
4506                                                 if( ctx )
4507                                                         osrfAppSessionStatus(
4508                                                                 ctx->session,
4509                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4510                                                                 "osrfMethodException",
4511                                                                 ctx->request,
4512                                                                 "Selected column may not be virtual in JSON query"
4513                                                         );
4514                                                 jsonIteratorFree( selclass_itr );
4515                                                 buffer_free( select_buf );
4516                                                 buffer_free( group_buf );
4517                                                 if( defaultselhash )
4518                                                         jsonObjectFree( defaultselhash );
4519                                                 free( join_clause );
4520                                                 return NULL;
4521                                         }
4522
4523                                         if( locale ) {
4524                                                 const char* i18n;
4525                                                 if( flags & DISABLE_I18N )
4526                                                         i18n = NULL;
4527                                                 else
4528                                                         i18n = osrfHashGet( field_def, "i18n" );
4529
4530                                                 if( str_is_true( i18n ) ) {
4531                                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
4532                                                                 "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
4533                                                                 class_tname, cname, col_name, class_pkey,
4534                                                                 cname, class_pkey, locale, col_name );
4535                                                 } else {
4536                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4537                                                                 cname, col_name, col_name );
4538                                                 }
4539                                         } else {
4540                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4541                                                                 cname, col_name, col_name );
4542                                         }
4543
4544                                 // ... but it could be an object, in which case we check for a Field Transform
4545                                 } else if( selfield->type == JSON_HASH ) {
4546
4547                                         const char* col_name = jsonObjectGetString(
4548                                                         jsonObjectGetKeyConst( selfield, "column" ) );
4549
4550                                         // Get the field definition from the IDL
4551                                         osrfHash* field_def = NULL;
4552                                         if (!osrfStringArrayContains(
4553                                                         osrfHashGet(
4554                                                                 osrfHashGet( class_field_set, col_name ),
4555                                                                 "suppress_controller"),
4556                                                         modulename
4557                                         ))
4558                                                 field_def = osrfHashGet( class_field_set, col_name );
4559
4560
4561                                         if( !field_def ) {
4562                                                 // No such field in current class
4563                                                 osrfLogError(
4564                                                         OSRF_LOG_MARK,
4565                                                         "%s: Selected column \"%s\" is not defined in IDL for class \"%s\"",
4566                                                         modulename,
4567                                                         col_name,
4568                                                         cname
4569                                                 );
4570                                                 if( ctx )
4571                                                         osrfAppSessionStatus(
4572                                                                 ctx->session,
4573                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4574                                                                 "osrfMethodException",
4575                                                                 ctx->request,
4576                                                                 "Selected column is not defined in JSON query"
4577                                                         );
4578                                                 jsonIteratorFree( selclass_itr );
4579                                                 buffer_free( select_buf );
4580                                                 buffer_free( group_buf );
4581                                                 if( defaultselhash )
4582                                                         jsonObjectFree( defaultselhash );
4583                                                 free( join_clause );
4584                                                 return NULL;
4585                                         } else if( str_is_true( osrfHashGet( field_def, "virtual" ))) {
4586                                                 // No such field in current class
4587                                                 osrfLogError(
4588                                                         OSRF_LOG_MARK,
4589                                                         "%s: Selected column \"%s\" is virtual for class \"%s\"",
4590                                                         modulename,
4591                                                         col_name,
4592                                                         cname
4593                                                 );
4594                                                 if( ctx )
4595                                                         osrfAppSessionStatus(
4596                                                                 ctx->session,
4597                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4598                                                                 "osrfMethodException",
4599                                                                 ctx->request,
4600                                                                 "Selected column is virtual in JSON query"
4601                                                         );
4602                                                 jsonIteratorFree( selclass_itr );
4603                                                 buffer_free( select_buf );
4604                                                 buffer_free( group_buf );
4605                                                 if( defaultselhash )
4606                                                         jsonObjectFree( defaultselhash );
4607                                                 free( join_clause );
4608                                                 return NULL;
4609                                         }
4610
4611                                         // Decide what to use as a column alias
4612                                         const char* _alias;
4613                                         if((tmp_const = jsonObjectGetKeyConst( selfield, "alias" ))) {
4614                                                 _alias = jsonObjectGetString( tmp_const );
4615                                         } else if((tmp_const = jsonObjectGetKeyConst( selfield, "result_field" ))) { // Use result_field name as the alias
4616                                                 _alias = jsonObjectGetString( tmp_const );
4617                                         } else {         // Use field name as the alias
4618                                                 _alias = col_name;
4619                                         }
4620
4621                                         if( jsonObjectGetKeyConst( selfield, "transform" )) {
4622                                                 char* transform_str = searchFieldTransform(
4623                                                         class_info->alias, field_def, selfield );
4624                                                 if( transform_str ) {
4625                                                         buffer_fadd( select_buf, " %s AS \"%s\"", transform_str, _alias );
4626                                                         free( transform_str );
4627                                                 } else {
4628                                                         if( ctx )
4629                                                                 osrfAppSessionStatus(
4630                                                                         ctx->session,
4631                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4632                                                                         "osrfMethodException",
4633                                                                         ctx->request,
4634                                                                         "Unable to generate transform function in JSON query"
4635                                                                 );
4636                                                         jsonIteratorFree( selclass_itr );
4637                                                         buffer_free( select_buf );
4638                                                         buffer_free( group_buf );
4639                                                         if( defaultselhash )
4640                                                                 jsonObjectFree( defaultselhash );
4641                                                         free( join_clause );
4642                                                         return NULL;
4643                                                 }
4644                                         } else {
4645
4646                                                 if( locale ) {
4647                                                         const char* i18n;
4648                                                         if( flags & DISABLE_I18N )
4649                                                                 i18n = NULL;
4650                                                         else
4651                                                                 i18n = osrfHashGet( field_def, "i18n" );
4652
4653                                                         if( str_is_true( i18n ) ) {
4654                                                                 buffer_fadd( select_buf,
4655                                                                         " oils_i18n_xlate('%s', '%s', '%s', '%s', "
4656                                                                         "\"%s\".%s::TEXT, '%s') AS \"%s\"",
4657                                                                         class_tname, cname, col_name, class_pkey, cname,
4658                                                                         class_pkey, locale, _alias );
4659                                                         } else {
4660                                                                 buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4661                                                                         cname, col_name, _alias );
4662                                                         }
4663                                                 } else {
4664                                                         buffer_fadd( select_buf, " \"%s\".%s AS \"%s\"",
4665                                                                 cname, col_name, _alias );
4666                                                 }
4667                                         }
4668                                 }
4669                                 else {
4670                                         osrfLogError(
4671                                                 OSRF_LOG_MARK,
4672                                                 "%s: Selected item is unexpected JSON type: %s",
4673                                                 modulename,
4674                                                 json_type( selfield->type )
4675                                         );
4676                                         if( ctx )
4677                                                 osrfAppSessionStatus(
4678                                                         ctx->session,
4679                                                         OSRF_STATUS_INTERNALSERVERERROR,
4680                                                         "osrfMethodException",
4681                                                         ctx->request,
4682                                                         "Ill-formed SELECT item in JSON query"
4683                                                 );
4684                                         jsonIteratorFree( selclass_itr );
4685                                         buffer_free( select_buf );
4686                                         buffer_free( group_buf );
4687                                         if( defaultselhash )
4688                                                 jsonObjectFree( defaultselhash );
4689                                         free( join_clause );
4690                                         return NULL;
4691                                 }
4692
4693                                 const jsonObject* agg_obj = jsonObjectGetKeyConst( selfield, "aggregate" );
4694                                 if( obj_is_true( agg_obj ) )
4695                                         aggregate_found = 1;
4696                                 else {
4697                                         // Append a comma (except for the first one)
4698                                         // and add the column to a GROUP BY clause
4699                                         if( gfirst )
4700                                                 gfirst = 0;
4701                                         else
4702                                                 OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4703
4704                                         buffer_fadd( group_buf, " %d", sel_pos );
4705                                 }
4706
4707 #if 0
4708                             if (is_agg->size || (flags & SELECT_DISTINCT)) {
4709
4710                                         const jsonObject* aggregate_obj = jsonObjectGetKeyConst( selfield, "aggregate");
4711                                     if ( ! obj_is_true( aggregate_obj ) ) {
4712                                             if (gfirst) {
4713                                                     gfirst = 0;
4714                                             } else {
4715                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4716                                             }
4717
4718                                             buffer_fadd(group_buf, " %d", sel_pos);
4719
4720                                         /*
4721                                     } else if (is_agg = jsonObjectGetKeyConst( selfield, "having" )) {
4722                                             if (gfirst) {
4723                                                     gfirst = 0;
4724                                             } else {
4725                                                         OSRF_BUFFER_ADD_CHAR( group_buf, ',' );
4726                                             }
4727
4728                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4729                                                 OSRF_BUFFER_ADD_CHAR(group_buf, ' ');
4730                                                 OSRF_BUFFER_ADD(group_buf, _column);
4731                                             _column = searchFieldTransform(class_info->alias, field, selfield);
4732                                         */
4733                                     }
4734                             }
4735 #endif
4736
4737                                 sel_pos++;
4738                         } // end while -- iterating across SELECT columns
4739
4740                 } // end while -- iterating across classes
4741
4742                 jsonIteratorFree( selclass_itr );
4743         }
4744
4745         char* col_list = buffer_release( select_buf );
4746
4747         // Make sure the SELECT list isn't empty.  This can happen, for example,
4748         // if we try to build a default SELECT clause from a non-core table.
4749
4750         if( ! *col_list ) {
4751                 osrfLogError( OSRF_LOG_MARK, "%s: SELECT clause is empty", modulename );
4752                 if( ctx )
4753                         osrfAppSessionStatus(
4754                                 ctx->session,
4755                                 OSRF_STATUS_INTERNALSERVERERROR,
4756                                 "osrfMethodException",
4757                                 ctx->request,
4758                                 "SELECT list is empty"
4759                 );
4760                 free( col_list );
4761                 buffer_free( group_buf );
4762                 if( defaultselhash )
4763                         jsonObjectFree( defaultselhash );
4764                 free( join_clause );
4765                 return NULL;
4766         }
4767
4768         char* table = NULL;
4769         if( from_function )
4770                 table = searchValueTransform( join_hash );
4771         else
4772                 table = strdup( curr_query->core.source_def );
4773
4774         if( !table ) {
4775                 if( ctx )
4776                         osrfAppSessionStatus(
4777                                 ctx->session,
4778                                 OSRF_STATUS_INTERNALSERVERERROR,
4779                                 "osrfMethodException",
4780                                 ctx->request,
4781                                 "Unable to identify table for core class"
4782                         );
4783                 free( col_list );
4784                 buffer_free( group_buf );
4785                 if( defaultselhash )
4786                         jsonObjectFree( defaultselhash );
4787                 free( join_clause );
4788                 return NULL;
4789         }
4790
4791         // Put it all together
4792         growing_buffer* sql_buf = buffer_init( 128 );
4793         buffer_fadd(sql_buf, "SELECT %s FROM %s AS \"%s\" ", col_list, table, core_class );
4794         free( col_list );
4795         free( table );
4796
4797         // Append the join clause, if any
4798         if( join_clause ) {
4799                 buffer_add(sql_buf, join_clause );
4800                 free( join_clause );
4801         }
4802
4803         char* order_by_list = NULL;
4804         char* having_buf = NULL;
4805
4806         if( !from_function ) {
4807
4808                 // Build a WHERE clause, if there is one
4809                 if( search_hash ) {
4810                         buffer_add( sql_buf, " WHERE " );
4811
4812                         // and it's on the WHERE clause
4813                         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
4814                         if( ! pred ) {
4815                                 if( ctx ) {
4816                                         osrfAppSessionStatus(
4817                                                 ctx->session,
4818                                                 OSRF_STATUS_INTERNALSERVERERROR,
4819                                                 "osrfMethodException",
4820                                                 ctx->request,
4821                                                 "Severe query error in WHERE predicate -- see error log for more details"
4822                                         );
4823                                 }
4824                                 buffer_free( group_buf );
4825                                 buffer_free( sql_buf );
4826                                 if( defaultselhash )
4827                                         jsonObjectFree( defaultselhash );
4828                                 return NULL;
4829                         }
4830
4831                         buffer_add( sql_buf, pred );
4832                         free( pred );
4833                 }
4834
4835                 // Build a HAVING clause, if there is one
4836                 if( having_hash ) {
4837
4838                         // and it's on the the WHERE clause
4839                         having_buf = searchWHERE( having_hash, &curr_query->core, AND_OP_JOIN, ctx );
4840
4841                         if( ! having_buf ) {
4842                                 if( ctx ) {
4843                                                 osrfAppSessionStatus(
4844                                                 ctx->session,
4845                                                 OSRF_STATUS_INTERNALSERVERERROR,
4846                                                 "osrfMethodException",
4847                                                 ctx->request,
4848                                                 "Severe query error in HAVING predicate -- see error log for more details"
4849                                         );
4850                                 }
4851                                 buffer_free( group_buf );
4852                                 buffer_free( sql_buf );
4853                                 if( defaultselhash )
4854                                         jsonObjectFree( defaultselhash );
4855                                 return NULL;
4856                         }
4857                 }
4858
4859                 // Build an ORDER BY clause, if there is one
4860                 if( NULL == order_hash )
4861                         ;  // No ORDER BY? do nothing
4862                 else if( JSON_ARRAY == order_hash->type ) {
4863                         order_by_list = buildOrderByFromArray( ctx, order_hash );
4864                         if( !order_by_list ) {
4865                                 free( having_buf );
4866                                 buffer_free( group_buf );
4867                                 buffer_free( sql_buf );
4868                                 if( defaultselhash )
4869                                         jsonObjectFree( defaultselhash );
4870                                 return NULL;
4871                         }
4872                 } else if( JSON_HASH == order_hash->type ) {
4873                         // This hash is keyed on class alias.  Each class has either
4874                         // an array of field names or a hash keyed on field name.
4875                         growing_buffer* order_buf = NULL;  // to collect ORDER BY list
4876                         jsonIterator* class_itr = jsonNewIterator( order_hash );
4877                         while( (snode = jsonIteratorNext( class_itr )) ) {
4878
4879                                 ClassInfo* order_class_info = search_alias( class_itr->key );
4880                                 if( ! order_class_info ) {
4881                                         osrfLogError( OSRF_LOG_MARK,
4882                                                 "%s: Invalid class \"%s\" referenced in ORDER BY clause",
4883                                                 modulename, class_itr->key );
4884                                         if( ctx )
4885                                                 osrfAppSessionStatus(
4886                                                         ctx->session,
4887                                                         OSRF_STATUS_INTERNALSERVERERROR,
4888                                                         "osrfMethodException",
4889                                                         ctx->request,
4890                                                         "Invalid class referenced in ORDER BY clause -- "
4891                                                                 "see error log for more details"
4892                                                 );
4893                                         jsonIteratorFree( class_itr );
4894                                         buffer_free( order_buf );
4895                                         free( having_buf );
4896                                         buffer_free( group_buf );
4897                                         buffer_free( sql_buf );
4898                                         if( defaultselhash )
4899                                                 jsonObjectFree( defaultselhash );
4900                                         return NULL;
4901                                 }
4902
4903                                 osrfHash* field_list_def = order_class_info->fields;
4904
4905                                 if( snode->type == JSON_HASH ) {
4906
4907                                         // Hash is keyed on field names from the current class.  For each field
4908                                         // there is another layer of hash to define the sorting details, if any,
4909                                         // or a string to indicate direction of sorting.
4910                                         jsonIterator* order_itr = jsonNewIterator( snode );
4911                                         while( (onode = jsonIteratorNext( order_itr )) ) {
4912
4913                                                 osrfHash* field_def = osrfHashGet( field_list_def, order_itr->key );
4914                                                 if( !field_def ) {
4915                                                         osrfLogError( OSRF_LOG_MARK,
4916                                                                 "%s: Invalid field \"%s\" in ORDER BY clause",
4917                                                                 modulename, order_itr->key );
4918                                                         if( ctx )
4919                                                                 osrfAppSessionStatus(
4920                                                                         ctx->session,
4921                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4922                                                                         "osrfMethodException",
4923                                                                         ctx->request,
4924                                                                         "Invalid field in ORDER BY clause -- "
4925                                                                         "see error log for more details"
4926                                                                 );
4927                                                         jsonIteratorFree( order_itr );
4928                                                         jsonIteratorFree( class_itr );
4929                                                         buffer_free( order_buf );
4930                                                         free( having_buf );
4931                                                         buffer_free( group_buf );
4932                                                         buffer_free( sql_buf );
4933                                                         if( defaultselhash )
4934                                                                 jsonObjectFree( defaultselhash );
4935                                                         return NULL;
4936                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
4937                                                         osrfLogError( OSRF_LOG_MARK,
4938                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
4939                                                                 modulename, order_itr->key );
4940                                                         if( ctx )
4941                                                                 osrfAppSessionStatus(
4942                                                                         ctx->session,
4943                                                                         OSRF_STATUS_INTERNALSERVERERROR,
4944                                                                         "osrfMethodException",
4945                                                                         ctx->request,
4946                                                                         "Virtual field in ORDER BY clause -- "
4947                                                                         "see error log for more details"
4948                                                         );
4949                                                         jsonIteratorFree( order_itr );
4950                                                         jsonIteratorFree( class_itr );
4951                                                         buffer_free( order_buf );
4952                                                         free( having_buf );
4953                                                         buffer_free( group_buf );
4954                                                         buffer_free( sql_buf );
4955                                                         if( defaultselhash )
4956                                                                 jsonObjectFree( defaultselhash );
4957                                                         return NULL;
4958                                                 }
4959
4960                                                 const char* direction = NULL;
4961                                                 if( onode->type == JSON_HASH ) {
4962                                                         if( jsonObjectGetKeyConst( onode, "transform" ) ) {
4963                                                                 string = searchFieldTransform(
4964                                                                         class_itr->key,
4965                                                                         osrfHashGet( field_list_def, order_itr->key ),
4966                                                                         onode
4967                                                                 );
4968                                                                 if( ! string ) {
4969                                                                         if( ctx ) osrfAppSessionStatus(
4970                                                                                 ctx->session,
4971                                                                                 OSRF_STATUS_INTERNALSERVERERROR,
4972                                                                                 "osrfMethodException",
4973                                                                                 ctx->request,
4974                                                                                 "Severe query error in ORDER BY clause -- "
4975                                                                                 "see error log for more details"
4976                                                                         );
4977                                                                         jsonIteratorFree( order_itr );
4978                                                                         jsonIteratorFree( class_itr );
4979                                                                         free( having_buf );
4980                                                                         buffer_free( group_buf );
4981                                                                         buffer_free( order_buf);
4982                                                                         buffer_free( sql_buf );
4983                                                                         if( defaultselhash )
4984                                                                                 jsonObjectFree( defaultselhash );
4985                                                                         return NULL;
4986                                                                 }
4987                                                         } else {
4988                                                                 growing_buffer* field_buf = buffer_init( 16 );
4989                                                                 buffer_fadd( field_buf, "\"%s\".%s",
4990                                                                         class_itr->key, order_itr->key );
4991                                                                 string = buffer_release( field_buf );
4992                                                         }
4993
4994                                                         if( (tmp_const = jsonObjectGetKeyConst( onode, "direction" )) ) {
4995                                                                 const char* dir = jsonObjectGetString( tmp_const );
4996                                                                 if(!strncasecmp( dir, "d", 1 )) {
4997                                                                         direction = " DESC";
4998                                                                 } else {
4999                                                                         direction = " ASC";
5000                                                                 }
5001                                                         }
5002
5003                                                 } else if( JSON_NULL == onode->type || JSON_ARRAY == onode->type ) {
5004                                                         osrfLogError( OSRF_LOG_MARK,
5005                                                                 "%s: Expected JSON_STRING in ORDER BY clause; found %s",
5006                                                                 modulename, json_type( onode->type ) );
5007                                                         if( ctx )
5008                                                                 osrfAppSessionStatus(
5009                                                                         ctx->session,
5010                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5011                                                                         "osrfMethodException",
5012                                                                         ctx->request,
5013                                                                         "Malformed ORDER BY clause -- see error log for more details"
5014                                                                 );
5015                                                         jsonIteratorFree( order_itr );
5016                                                         jsonIteratorFree( class_itr );
5017                                                         free( having_buf );
5018                                                         buffer_free( group_buf );
5019                                                         buffer_free( order_buf );
5020                                                         buffer_free( sql_buf );
5021                                                         if( defaultselhash )
5022                                                                 jsonObjectFree( defaultselhash );
5023                                                         return NULL;
5024
5025                                                 } else {
5026                                                         string = strdup( order_itr->key );
5027                                                         const char* dir = jsonObjectGetString( onode );
5028                                                         if( !strncasecmp( dir, "d", 1 )) {
5029                                                                 direction = " DESC";
5030                                                         } else {
5031                                                                 direction = " ASC";
5032                                                         }
5033                                                 }
5034
5035                                                 if( order_buf )
5036                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5037                                                 else
5038                                                         order_buf = buffer_init( 128 );
5039
5040                                                 OSRF_BUFFER_ADD( order_buf, string );
5041                                                 free( string );
5042
5043                                                 if( direction ) {
5044                                                          OSRF_BUFFER_ADD( order_buf, direction );
5045                                                 }
5046
5047                                         } // end while
5048                                         jsonIteratorFree( order_itr );
5049
5050                                 } else if( snode->type == JSON_ARRAY ) {
5051
5052                                         // Array is a list of fields from the current class
5053                                         unsigned long order_idx = 0;
5054                                         while(( onode = jsonObjectGetIndex( snode, order_idx++ ) )) {
5055
5056                                                 const char* _f = jsonObjectGetString( onode );
5057
5058                                                 osrfHash* field_def = osrfHashGet( field_list_def, _f );
5059                                                 if( !field_def ) {
5060                                                         osrfLogError( OSRF_LOG_MARK,
5061                                                                         "%s: Invalid field \"%s\" in ORDER BY clause",
5062                                                                         modulename, _f );
5063                                                         if( ctx )
5064                                                                 osrfAppSessionStatus(
5065                                                                         ctx->session,
5066                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5067                                                                         "osrfMethodException",
5068                                                                         ctx->request,
5069                                                                         "Invalid field in ORDER BY clause -- "
5070                                                                         "see error log for more details"
5071                                                                 );
5072                                                         jsonIteratorFree( class_itr );
5073                                                         buffer_free( order_buf );
5074                                                         free( having_buf );
5075                                                         buffer_free( group_buf );
5076                                                         buffer_free( sql_buf );
5077                                                         if( defaultselhash )
5078                                                                 jsonObjectFree( defaultselhash );
5079                                                         return NULL;
5080                                                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5081                                                         osrfLogError( OSRF_LOG_MARK,
5082                                                                 "%s: Virtual field \"%s\" in ORDER BY clause",
5083                                                                 modulename, _f );
5084                                                         if( ctx )
5085                                                                 osrfAppSessionStatus(
5086                                                                         ctx->session,
5087                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5088                                                                         "osrfMethodException",
5089                                                                         ctx->request,
5090                                                                         "Virtual field in ORDER BY clause -- "
5091                                                                         "see error log for more details"
5092                                                                 );
5093                                                         jsonIteratorFree( class_itr );
5094                                                         buffer_free( order_buf );
5095                                                         free( having_buf );
5096                                                         buffer_free( group_buf );
5097                                                         buffer_free( sql_buf );
5098                                                         if( defaultselhash )
5099                                                                 jsonObjectFree( defaultselhash );
5100                                                         return NULL;
5101                                                 }
5102
5103                                                 if( order_buf )
5104                                                         OSRF_BUFFER_ADD( order_buf, ", " );
5105                                                 else
5106                                                         order_buf = buffer_init( 128 );
5107
5108                                                 buffer_fadd( order_buf, "\"%s\".%s", class_itr->key, _f );
5109
5110                                         } // end while
5111
5112                                 // IT'S THE OOOOOOOOOOOLD STYLE!
5113                                 } else {
5114                                         osrfLogError( OSRF_LOG_MARK,
5115                                                 "%s: Possible SQL injection attempt; direct order by is not allowed",
5116                                                 modulename );
5117                                         if(ctx) {
5118                                                 osrfAppSessionStatus(
5119                                                         ctx->session,
5120                                                         OSRF_STATUS_INTERNALSERVERERROR,
5121                                                         "osrfMethodException",
5122                                                         ctx->request,
5123                                                         "Severe query error -- see error log for more details"
5124                                                 );
5125                                         }
5126
5127                                         free( having_buf );
5128                                         buffer_free( group_buf );
5129                                         buffer_free( order_buf );
5130                                         buffer_free( sql_buf );
5131                                         if( defaultselhash )
5132                                                 jsonObjectFree( defaultselhash );
5133                                         jsonIteratorFree( class_itr );
5134                                         return NULL;
5135                                 }
5136                         } // end while
5137                         jsonIteratorFree( class_itr );
5138                         if( order_buf )
5139                                 order_by_list = buffer_release( order_buf );
5140                 } else {
5141                         osrfLogError( OSRF_LOG_MARK,
5142                                 "%s: Malformed ORDER BY clause; expected JSON_HASH or JSON_ARRAY, found %s",
5143                                 modulename, json_type( order_hash->type ) );
5144                         if( ctx )
5145                                 osrfAppSessionStatus(
5146                                         ctx->session,
5147                                         OSRF_STATUS_INTERNALSERVERERROR,
5148                                         "osrfMethodException",
5149                                         ctx->request,
5150                                         "Malformed ORDER BY clause -- see error log for more details"
5151                                 );
5152                         free( having_buf );
5153                         buffer_free( group_buf );
5154                         buffer_free( sql_buf );
5155                         if( defaultselhash )
5156                                 jsonObjectFree( defaultselhash );
5157                         return NULL;
5158                 }
5159         }
5160
5161         string = buffer_release( group_buf );
5162
5163         if( *string && ( aggregate_found || (flags & SELECT_DISTINCT) ) ) {
5164                 OSRF_BUFFER_ADD( sql_buf, " GROUP BY " );
5165                 OSRF_BUFFER_ADD( sql_buf, string );
5166         }
5167
5168         free( string );
5169
5170         if( having_buf && *having_buf ) {
5171                 OSRF_BUFFER_ADD( sql_buf, " HAVING " );
5172                 OSRF_BUFFER_ADD( sql_buf, having_buf );
5173                 free( having_buf );
5174         }
5175
5176         if( order_by_list ) {
5177
5178                 if( *order_by_list ) {
5179                         OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5180                         OSRF_BUFFER_ADD( sql_buf, order_by_list );
5181                 }
5182
5183                 free( order_by_list );
5184         }
5185
5186         if( limit ){
5187                 const char* str = jsonObjectGetString( limit );
5188                 if (str) { // limit could be JSON_NULL, etc.
5189                         buffer_fadd( sql_buf, " LIMIT %d", atoi( str ));
5190                 }
5191         }
5192
5193         if( offset ) {
5194                 const char* str = jsonObjectGetString( offset );
5195                 if (str) {
5196                         buffer_fadd( sql_buf, " OFFSET %d", atoi( str ));
5197                 }
5198         }
5199
5200         if( !(flags & SUBSELECT) )
5201                 OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5202
5203         if( defaultselhash )
5204                  jsonObjectFree( defaultselhash );
5205
5206         return buffer_release( sql_buf );
5207
5208 } // end of SELECT()
5209
5210 /**
5211         @brief Build a list of ORDER BY expressions.
5212         @param ctx Pointer to the method context.
5213         @param order_array Pointer to a JSON_ARRAY of field specifications.
5214         @return Pointer to a string containing a comma-separated list of ORDER BY expressions.
5215         Each expression may be either a column reference or a function call whose first parameter
5216         is a column reference.
5217
5218         Each entry in @a order_array must be a JSON_HASH with values for "class" and "field".
5219         It may optionally include entries for "direction" and/or "transform".
5220
5221         The calling code is responsible for freeing the returned string.
5222 */
5223 static char* buildOrderByFromArray( osrfMethodContext* ctx, const jsonObject* order_array ) {
5224         if( ! order_array ) {
5225                 osrfLogError( OSRF_LOG_MARK, "%s: Logic error: NULL pointer for ORDER BY clause",
5226                         modulename );
5227                 if( ctx )
5228                         osrfAppSessionStatus(
5229                                 ctx->session,
5230                                 OSRF_STATUS_INTERNALSERVERERROR,
5231                                 "osrfMethodException",
5232                                 ctx->request,
5233                                 "Logic error: ORDER BY clause expected, not found; "
5234                                         "see error log for more details"
5235                         );
5236                 return NULL;
5237         } else if( order_array->type != JSON_ARRAY ) {
5238                 osrfLogError( OSRF_LOG_MARK,
5239                         "%s: Logic error: Expected JSON_ARRAY for ORDER BY clause, not found", modulename );
5240                 if( ctx )
5241                         osrfAppSessionStatus(
5242                         ctx->session,
5243                         OSRF_STATUS_INTERNALSERVERERROR,
5244                         "osrfMethodException",
5245                         ctx->request,
5246                         "Logic error: Unexpected format for ORDER BY clause; see error log for more details" );
5247                 return NULL;
5248         }
5249
5250         growing_buffer* order_buf = buffer_init( 128 );
5251         int first = 1;        // boolean
5252         int order_idx = 0;
5253         jsonObject* order_spec;
5254         while( (order_spec = jsonObjectGetIndex( order_array, order_idx++ ))) {
5255
5256                 if( JSON_HASH != order_spec->type ) {
5257                         osrfLogError( OSRF_LOG_MARK,
5258                                 "%s: Malformed field specification in ORDER BY clause; "
5259                                 "expected JSON_HASH, found %s",
5260                                 modulename, json_type( order_spec->type ) );
5261                         if( ctx )
5262                                 osrfAppSessionStatus(
5263                                          ctx->session,
5264                                         OSRF_STATUS_INTERNALSERVERERROR,
5265                                         "osrfMethodException",
5266                                         ctx->request,
5267                                         "Malformed ORDER BY clause -- see error log for more details"
5268                                 );
5269                         buffer_free( order_buf );
5270                         return NULL;
5271                 }
5272
5273                 const char* class_alias =
5274                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "class" ));
5275                 const char* field =
5276                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "field" ));
5277
5278                 jsonObject* compare_to = jsonObjectGetKey( order_spec, "compare" );
5279
5280                 if( !field || !class_alias ) {
5281                         osrfLogError( OSRF_LOG_MARK,
5282                                 "%s: Missing class or field name in field specification of ORDER BY clause",
5283                                 modulename );
5284                         if( ctx )
5285                                 osrfAppSessionStatus(
5286                                         ctx->session,
5287                                         OSRF_STATUS_INTERNALSERVERERROR,
5288                                         "osrfMethodException",
5289                                         ctx->request,
5290                                         "Malformed ORDER BY clause -- see error log for more details"
5291                                 );
5292                         buffer_free( order_buf );
5293                         return NULL;
5294                 }
5295
5296                 const ClassInfo* order_class_info = search_alias( class_alias );
5297                 if( ! order_class_info ) {
5298                         osrfLogInternal( OSRF_LOG_MARK, "%s: ORDER BY clause references class \"%s\" "
5299                                 "not in FROM clause, skipping it", modulename, class_alias );
5300                         continue;
5301                 }
5302
5303                 // Add a separating comma, except at the beginning
5304                 if( first )
5305                         first = 0;
5306                 else
5307                         OSRF_BUFFER_ADD( order_buf, ", " );
5308
5309                 osrfHash* field_def = osrfHashGet( order_class_info->fields, field );
5310                 if( !field_def ) {
5311                         osrfLogError( OSRF_LOG_MARK,
5312                                 "%s: Invalid field \"%s\".%s referenced in ORDER BY clause",
5313                                 modulename, class_alias, field );
5314                         if( ctx )
5315                                 osrfAppSessionStatus(
5316                                         ctx->session,
5317                                         OSRF_STATUS_INTERNALSERVERERROR,
5318                                         "osrfMethodException",
5319                                         ctx->request,
5320                                         "Invalid field referenced in ORDER BY clause -- "
5321                                         "see error log for more details"
5322                                 );
5323                         free( order_buf );
5324                         return NULL;
5325                 } else if( str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5326                         osrfLogError( OSRF_LOG_MARK, "%s: Virtual field \"%s\" in ORDER BY clause",
5327                                 modulename, field );
5328                         if( ctx )
5329                                 osrfAppSessionStatus(
5330                                         ctx->session,
5331                                         OSRF_STATUS_INTERNALSERVERERROR,
5332                                         "osrfMethodException",
5333                                         ctx->request,
5334                                         "Virtual field in ORDER BY clause -- see error log for more details"
5335                                 );
5336                         buffer_free( order_buf );
5337                         return NULL;
5338                 }
5339
5340                 if( jsonObjectGetKeyConst( order_spec, "transform" )) {
5341                         char* transform_str = searchFieldTransform( class_alias, field_def, order_spec );
5342                         if( ! transform_str ) {
5343                                 if( ctx )
5344                                         osrfAppSessionStatus(
5345                                                 ctx->session,
5346                                                 OSRF_STATUS_INTERNALSERVERERROR,
5347                                                 "osrfMethodException",
5348                                                 ctx->request,
5349                                                 "Severe query error in ORDER BY clause -- "
5350                                                 "see error log for more details"
5351                                         );
5352                                 buffer_free( order_buf );
5353                                 return NULL;
5354                         }
5355
5356                         OSRF_BUFFER_ADD( order_buf, transform_str );
5357                         free( transform_str );
5358                 } else if( compare_to ) {
5359                         char* compare_str = searchPredicate( order_class_info, field_def, compare_to, ctx );
5360                         if( ! compare_str ) {
5361                                 if( ctx )
5362                                         osrfAppSessionStatus(
5363                                                 ctx->session,
5364                                                 OSRF_STATUS_INTERNALSERVERERROR,
5365                                                 "osrfMethodException",
5366                                                 ctx->request,
5367                                                 "Severe query error in ORDER BY clause -- "
5368                                                 "see error log for more details"
5369                                         );
5370                                 buffer_free( order_buf );
5371                                 return NULL;
5372                         }
5373
5374                         buffer_fadd( order_buf, "(%s)", compare_str );
5375                         free( compare_str );
5376                 }
5377                 else
5378                         buffer_fadd( order_buf, "\"%s\".%s", class_alias, field );
5379
5380                 const char* direction =
5381                         jsonObjectGetString( jsonObjectGetKeyConst( order_spec, "direction" ) );
5382                 if( direction ) {
5383                         if( direction[ 0 ] && ( 'd' == direction[ 0 ] || 'D' == direction[ 0 ] ) )
5384                                 OSRF_BUFFER_ADD( order_buf, " DESC" );
5385                         else
5386                                 OSRF_BUFFER_ADD( order_buf, " ASC" );
5387                 }
5388         }
5389
5390         return buffer_release( order_buf );
5391 }
5392
5393 /**
5394         @brief Build a SELECT statement.
5395         @param search_hash Pointer to a JSON_HASH or JSON_ARRAY encoding the WHERE clause.
5396         @param rest_of_query Pointer to a JSON_HASH containing any other SQL clauses.
5397         @param meta Pointer to the class metadata for the core class.
5398         @param ctx Pointer to the method context.
5399         @return Pointer to a character string containing the WHERE clause; or NULL upon error.
5400
5401         Within the rest_of_query hash, the meaningful keys are "join", "select", "no_i18n",
5402         "order_by", "limit", and "offset".
5403
5404         The SELECT statements built here are distinct from those built for the json_query method.
5405 */
5406 static char* buildSELECT ( const jsonObject* search_hash, jsonObject* rest_of_query,
5407         osrfHash* meta, osrfMethodContext* ctx ) {
5408
5409         const char* locale = osrf_message_get_last_locale();
5410
5411         osrfHash* fields = osrfHashGet( meta, "fields" );
5412         const char* core_class = osrfHashGet( meta, "classname" );
5413
5414         const jsonObject* join_hash = jsonObjectGetKeyConst( rest_of_query, "join" );
5415
5416         jsonObject* selhash = NULL;
5417         jsonObject* defaultselhash = NULL;
5418
5419         growing_buffer* sql_buf = buffer_init( 128 );
5420         growing_buffer* select_buf = buffer_init( 128 );
5421
5422         if( !(selhash = jsonObjectGetKey( rest_of_query, "select" )) ) {
5423                 defaultselhash = jsonNewObjectType( JSON_HASH );
5424                 selhash = defaultselhash;
5425         }
5426
5427         // If there's no SELECT list for the core class, build one
5428         if( !jsonObjectGetKeyConst( selhash, core_class ) ) {
5429                 jsonObject* field_list = jsonNewObjectType( JSON_ARRAY );
5430
5431                 // Add every non-virtual field to the field list
5432                 osrfHash* field_def = NULL;
5433                 osrfHashIterator* field_itr = osrfNewHashIterator( fields );
5434                 while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
5435                         if( ! str_is_true( osrfHashGet( field_def, "virtual" ) ) ) {
5436                                 const char* field = osrfHashIteratorKey( field_itr );
5437                                 jsonObjectPush( field_list, jsonNewObject( field ) );
5438                         }
5439                 }
5440                 osrfHashIteratorFree( field_itr );
5441                 jsonObjectSetKey( selhash, core_class, field_list );
5442         }
5443
5444         // Build a list of columns for the SELECT clause
5445         int first = 1;
5446         const jsonObject* snode = NULL;
5447         jsonIterator* class_itr = jsonNewIterator( selhash );
5448         while( (snode = jsonIteratorNext( class_itr )) ) {        // For each class
5449
5450                 // If the class isn't in the IDL, ignore it
5451                 const char* cname = class_itr->key;
5452                 osrfHash* idlClass = osrfHashGet( oilsIDL(), cname );
5453                 if( !idlClass )
5454                         continue;
5455
5456                 // If the class isn't the core class, and isn't in the JOIN clause, ignore it
5457                 if( strcmp( core_class, class_itr->key )) {
5458                         if( !join_hash )
5459                                 continue;
5460
5461                         jsonObject* found = jsonObjectFindPath( join_hash, "//%s", class_itr->key );
5462                         if( !found->size ) {
5463                                 jsonObjectFree( found );
5464                                 continue;
5465                         }
5466
5467                         jsonObjectFree( found );
5468                 }
5469
5470                 const jsonObject* node = NULL;
5471                 jsonIterator* select_itr = jsonNewIterator( snode );
5472                 while( (node = jsonIteratorNext( select_itr )) ) {
5473                         const char* item_str = jsonObjectGetString( node );
5474                         osrfHash* field = osrfHashGet( osrfHashGet( idlClass, "fields" ), item_str );
5475                         char* fname = osrfHashGet( field, "name" );
5476
5477                         if( !field )
5478                                 continue;
5479
5480                         if (osrfStringArrayContains( osrfHashGet(field, "suppress_controller"), modulename ))
5481                                 continue;
5482
5483                         if( first ) {
5484                                 first = 0;
5485                         } else {
5486                                 OSRF_BUFFER_ADD_CHAR( select_buf, ',' );
5487                         }
5488
5489                         if( locale ) {
5490                                 const char* i18n;
5491                                 const jsonObject* no_i18n_obj = jsonObjectGetKeyConst( rest_of_query, "no_i18n" );
5492                                 if( obj_is_true( no_i18n_obj ) )    // Suppress internationalization?
5493                                         i18n = NULL;
5494                                 else
5495                                         i18n = osrfHashGet( field, "i18n" );
5496
5497                                 if( str_is_true( i18n ) ) {
5498                                         char* pkey = osrfHashGet( idlClass, "primarykey" );
5499                                         char* tname = osrfHashGet( idlClass, "tablename" );
5500
5501                                         buffer_fadd( select_buf, " oils_i18n_xlate('%s', '%s', '%s', "
5502                                                         "'%s', \"%s\".%s::TEXT, '%s') AS \"%s\"",
5503                                                         tname, cname, fname, pkey, cname, pkey, locale, fname );
5504                                 } else {
5505                                         buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5506                                 }
5507                         } else {
5508                                 buffer_fadd( select_buf, " \"%s\".%s", cname, fname );
5509                         }
5510                 }
5511
5512                 jsonIteratorFree( select_itr );
5513         }
5514
5515         jsonIteratorFree( class_itr );
5516
5517         char* col_list = buffer_release( select_buf );
5518         char* table = oilsGetRelation( meta );
5519         if( !table )
5520                 table = strdup( "(null)" );
5521
5522         buffer_fadd( sql_buf, "SELECT %s FROM %s AS \"%s\"", col_list, table, core_class );
5523         free( col_list );
5524         free( table );
5525
5526         // Clear the query stack (as a fail-safe precaution against possible
5527         // leftover garbage); then push the first query frame onto the stack.
5528         clear_query_stack();
5529         push_query_frame();
5530         if( add_query_core( NULL, core_class ) ) {
5531                 if( ctx )
5532                         osrfAppSessionStatus(
5533                                 ctx->session,
5534                                 OSRF_STATUS_INTERNALSERVERERROR,
5535                                 "osrfMethodException",
5536                                 ctx->request,
5537                                 "Unable to build query frame for core class"
5538                         );
5539                 buffer_free( sql_buf );
5540                 if( defaultselhash )
5541                         jsonObjectFree( defaultselhash );
5542                 return NULL;
5543         }
5544
5545         // Add the JOIN clauses, if any
5546         if( join_hash ) {
5547                 char* join_clause = searchJOIN( join_hash, &curr_query->core );
5548                 OSRF_BUFFER_ADD_CHAR( sql_buf, ' ' );
5549                 OSRF_BUFFER_ADD( sql_buf, join_clause );
5550                 free( join_clause );
5551         }
5552
5553         osrfLogDebug( OSRF_LOG_MARK, "%s pre-predicate SQL =  %s",
5554                 modulename, OSRF_BUFFER_C_STR( sql_buf ));
5555
5556         OSRF_BUFFER_ADD( sql_buf, " WHERE " );
5557
5558         // Add the conditions in the WHERE clause
5559         char* pred = searchWHERE( search_hash, &curr_query->core, AND_OP_JOIN, ctx );
5560         if( !pred ) {
5561                 osrfAppSessionStatus(
5562                         ctx->session,
5563                         OSRF_STATUS_INTERNALSERVERERROR,
5564                                 "osrfMethodException",
5565                                 ctx->request,
5566                                 "Severe query error -- see error log for more details"
5567                         );
5568                 buffer_free( sql_buf );
5569                 if( defaultselhash )
5570                         jsonObjectFree( defaultselhash );
5571                 clear_query_stack();
5572                 return NULL;
5573         } else {
5574                 buffer_add( sql_buf, pred );
5575                 free( pred );
5576         }
5577
5578         // Add the ORDER BY, LIMIT, and/or OFFSET clauses, if present
5579         if( rest_of_query ) {
5580                 const jsonObject* order_by = NULL;
5581                 if( ( order_by = jsonObjectGetKeyConst( rest_of_query, "order_by" )) ){
5582
5583                         char* order_by_list = NULL;
5584
5585                         if( JSON_ARRAY == order_by->type ) {
5586                                 order_by_list = buildOrderByFromArray( ctx, order_by );
5587                                 if( !order_by_list ) {
5588                                         buffer_free( sql_buf );
5589                                         if( defaultselhash )
5590                                                 jsonObjectFree( defaultselhash );
5591                                         clear_query_stack();
5592                                         return NULL;
5593                                 }
5594                         } else if( JSON_HASH == order_by->type ) {
5595                                 // We expect order_by to be a JSON_HASH keyed on class names.  Traverse it
5596                                 // and build a list of ORDER BY expressions.
5597                                 growing_buffer* order_buf = buffer_init( 128 );
5598                                 first = 1;
5599                                 jsonIterator* class_itr = jsonNewIterator( order_by );
5600                                 while( (snode = jsonIteratorNext( class_itr )) ) {  // For each class:
5601
5602                                         ClassInfo* order_class_info = search_alias( class_itr->key );
5603                                         if( ! order_class_info )
5604                                                 continue;    // class not referenced by FROM clause?  Ignore it.
5605
5606                                         if( JSON_HASH == snode->type ) {
5607
5608                                                 // If the data for the current class is a JSON_HASH, then it is
5609                                                 // keyed on field name.
5610
5611                                                 const jsonObject* onode = NULL;
5612                                                 jsonIterator* order_itr = jsonNewIterator( snode );
5613                                                 while( (onode = jsonIteratorNext( order_itr )) ) {  // For each field
5614
5615                                                         osrfHash* field_def = osrfHashGet(
5616                                                                 order_class_info->fields, order_itr->key );
5617                                                         if( !field_def )
5618                                                                 continue;    // Field not defined in IDL?  Ignore it.
5619                                                         if( str_is_true( osrfHashGet( field_def, "virtual")))
5620                                                                 continue;    // Field is virtual?  Ignore it.
5621
5622                                                         char* field_str = NULL;
5623                                                         char* direction = NULL;
5624                                                         if( onode->type == JSON_HASH ) {
5625                                                                 if( jsonObjectGetKeyConst( onode, "transform" ) ) {
5626                                                                         field_str = searchFieldTransform(
5627                                                                                 class_itr->key, field_def, onode );
5628                                                                         if( ! field_str ) {
5629                                                                                 osrfAppSessionStatus(
5630                                                                                         ctx->session,
5631                                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5632                                                                                         "osrfMethodException",
5633                                                                                         ctx->request,
5634                                                                                         "Severe query error in ORDER BY clause -- "
5635                                                                                         "see error log for more details"
5636                                                                                 );
5637                                                                                 jsonIteratorFree( order_itr );
5638                                                                                 jsonIteratorFree( class_itr );
5639                                                                                 buffer_free( order_buf );
5640                                                                                 buffer_free( sql_buf );
5641                                                                                 if( defaultselhash )
5642                                                                                         jsonObjectFree( defaultselhash );
5643                                                                                 clear_query_stack();
5644                                                                                 return NULL;
5645                                                                         }
5646                                                                 } else {
5647                                                                         growing_buffer* field_buf = buffer_init( 16 );
5648                                                                         buffer_fadd( field_buf, "\"%s\".%s",
5649                                                                                 class_itr->key, order_itr->key );
5650                                                                         field_str = buffer_release( field_buf );
5651                                                                 }
5652
5653                                                                 if( ( order_by = jsonObjectGetKeyConst( onode, "direction" )) ) {
5654                                                                         const char* dir = jsonObjectGetString( order_by );
5655                                                                         if(!strncasecmp( dir, "d", 1 )) {
5656                                                                                 direction = " DESC";
5657                                                                         }
5658                                                                 }
5659                                                         } else {
5660                                                                 field_str = strdup( order_itr->key );
5661                                                                 const char* dir = jsonObjectGetString( onode );
5662                                                                 if( !strncasecmp( dir, "d", 1 )) {
5663                                                                         direction = " DESC";
5664                                                                 } else {
5665                                                                         direction = " ASC";
5666                                                                 }
5667                                                         }
5668
5669                                                         if( first ) {
5670                                                                 first = 0;
5671                                                         } else {
5672                                                                 buffer_add( order_buf, ", " );
5673                                                         }
5674
5675                                                         buffer_add( order_buf, field_str );
5676                                                         free( field_str );
5677
5678                                                         if( direction ) {
5679                                                                 buffer_add( order_buf, direction );
5680                                                         }
5681                                                 } // end while; looping over ORDER BY expressions
5682
5683                                                 jsonIteratorFree( order_itr );
5684
5685                                         } else if( JSON_STRING == snode->type ) {
5686                                                 // We expect a comma-separated list of sort fields.
5687                                                 const char* str = jsonObjectGetString( snode );
5688                                                 if( strchr( str, ';' )) {
5689                                                         // No semicolons allowed.  It is theoretically possible for a
5690                                                         // legitimate semicolon to occur within quotes, but it's not likely
5691                                                         // to occur in practice in the context of an ORDER BY list.
5692                                                         osrfLogError( OSRF_LOG_MARK, "%s: Possible attempt at SOL injection -- "
5693                                                                 "semicolon found in ORDER BY list: \"%s\"", modulename, str );
5694                                                         if( ctx ) {
5695                                                                 osrfAppSessionStatus(
5696                                                                         ctx->session,
5697                                                                         OSRF_STATUS_INTERNALSERVERERROR,
5698                                                                         "osrfMethodException",
5699                                                                         ctx->request,
5700                                                                         "Possible attempt at SOL injection -- "
5701                                                                                 "semicolon found in ORDER BY list"
5702                                                                 );
5703                                                         }
5704                                                         jsonIteratorFree( class_itr );
5705                                                         buffer_free( order_buf );
5706                                                         buffer_free( sql_buf );
5707                                                         if( defaultselhash )
5708                                                                 jsonObjectFree( defaultselhash );
5709                                                         clear_query_stack();
5710                                                         return NULL;
5711                                                 }
5712                                                 buffer_add( order_buf, str );
5713                                                 break;
5714                                         }
5715
5716                                 } // end while; looping over order_by classes
5717
5718                                 jsonIteratorFree( class_itr );
5719                                 order_by_list = buffer_release( order_buf );
5720
5721                         } else {
5722                                 osrfLogWarning( OSRF_LOG_MARK,
5723                                         "\"order_by\" object in a query is not a JSON_HASH or JSON_ARRAY;"
5724                                         "no ORDER BY generated" );
5725                         }
5726
5727                         if( order_by_list && *order_by_list ) {
5728                                 OSRF_BUFFER_ADD( sql_buf, " ORDER BY " );
5729                                 OSRF_BUFFER_ADD( sql_buf, order_by_list );
5730                         }
5731
5732                         free( order_by_list );
5733                 }
5734
5735                 const jsonObject* limit = jsonObjectGetKeyConst( rest_of_query, "limit" );
5736                 if( limit ) {
5737                         const char* str = jsonObjectGetString( limit );
5738                         if (str) {
5739                                 buffer_fadd(
5740                                         sql_buf,
5741                                         " LIMIT %d",
5742                                         atoi(str)
5743                                 );
5744                         }
5745                 }
5746
5747                 const jsonObject* offset = jsonObjectGetKeyConst( rest_of_query, "offset" );
5748                 if( offset ) {
5749                         const char* str = jsonObjectGetString( offset );
5750                         if (str) {
5751                                 buffer_fadd(
5752                                         sql_buf,
5753                                         " OFFSET %d",
5754                                         atoi( str )
5755                                 );
5756                         }
5757                 }
5758         }
5759
5760         if( defaultselhash )
5761                 jsonObjectFree( defaultselhash );
5762         clear_query_stack();
5763
5764         OSRF_BUFFER_ADD_CHAR( sql_buf, ';' );
5765         return buffer_release( sql_buf );
5766 }
5767
5768 int doJSONSearch ( osrfMethodContext* ctx ) {
5769         if(osrfMethodVerifyContext( ctx )) {
5770                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
5771                 return -1;
5772         }
5773
5774         osrfLogDebug( OSRF_LOG_MARK, "Received query request" );
5775
5776         int err = 0;
5777
5778         jsonObject* hash = jsonObjectGetIndex( ctx->params, 0 );
5779
5780         int flags = 0;
5781
5782         if( obj_is_true( jsonObjectGetKeyConst( hash, "distinct" )))
5783                 flags |= SELECT_DISTINCT;
5784
5785         if( obj_is_true( jsonObjectGetKeyConst( hash, "no_i18n" )))
5786                 flags |= DISABLE_I18N;
5787
5788         osrfLogDebug( OSRF_LOG_MARK, "Building SQL ..." );
5789         clear_query_stack();       // a possibly needless precaution
5790         char* sql = buildQuery( ctx, hash, flags );
5791         clear_query_stack();
5792
5793         if( !sql ) {
5794                 err = -1;
5795                 return err;
5796         }
5797
5798         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
5799
5800         // XXX for now...
5801         dbhandle = writehandle;
5802
5803         dbi_result result = dbi_conn_query( dbhandle, sql );
5804
5805         if( result ) {
5806                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
5807
5808                 if( dbi_result_first_row( result )) {
5809                         /* JSONify the result */
5810                         osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
5811
5812                         do {
5813                                 jsonObject* return_val = oilsMakeJSONFromResult( result );
5814                                 osrfAppRespond( ctx, return_val );
5815                                 jsonObjectFree( return_val );
5816                         } while( dbi_result_next_row( result ));
5817
5818                 } else {
5819                         osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s", modulename, sql );
5820                 }
5821
5822                 osrfAppRespondComplete( ctx, NULL );
5823
5824                 /* clean up the query */
5825                 dbi_result_free( result );
5826
5827         } else {
5828                 err = -1;
5829                 const char* msg;
5830                 int errnum = dbi_conn_error( dbhandle, &msg );
5831                 osrfLogError( OSRF_LOG_MARK, "%s: Error with query [%s]: %d %s",
5832                         modulename, sql, errnum, msg ? msg : "(No description available)" );
5833                 osrfAppSessionStatus(
5834                         ctx->session,
5835                         OSRF_STATUS_INTERNALSERVERERROR,
5836                         "osrfMethodException",
5837                         ctx->request,
5838                         "Severe query error -- see error log for more details"
5839                 );
5840                 if( !oilsIsDBConnected( dbhandle ))
5841                         osrfAppSessionPanic( ctx->session );
5842         }
5843
5844         free( sql );
5845         return err;
5846 }
5847
5848 // The last parameter, err, is used to report an error condition by updating an int owned by
5849 // the calling code.
5850
5851 // In case of an error, we set *err to -1.  If there is no error, *err is left unchanged.
5852 // It is the responsibility of the calling code to initialize *err before the
5853 // call, so that it will be able to make sense of the result.
5854
5855 // Note also that we return NULL if and only if we set *err to -1.  So the err parameter is
5856 // redundant anyway.
5857 static jsonObject* doFieldmapperSearch( osrfMethodContext* ctx, osrfHash* class_meta,
5858                 jsonObject* where_hash, jsonObject* query_hash, int* err ) {
5859
5860         const char* tz = _sanitize_tz_name(ctx->session->session_tz);
5861
5862         // XXX for now...
5863         dbhandle = writehandle;
5864
5865         char* core_class = osrfHashGet( class_meta, "classname" );
5866         osrfLogDebug( OSRF_LOG_MARK, "entering doFieldmapperSearch() with core_class %s", core_class );
5867
5868         char* pkey = osrfHashGet( class_meta, "primarykey" );
5869
5870         if (!ctx->session->userData)
5871                 (void) initSessionCache( ctx );
5872
5873         char *methodtype = osrfHashGet( (osrfHash *) ctx->method->userData, "methodtype" );
5874         char *inside_verify = osrfHashGet( (osrfHash*) ctx->session->userData, "inside_verify" );
5875         int need_to_verify = (inside_verify ? !atoi(inside_verify) : 1);
5876
5877         int i_respond_directly = 0;
5878         int flesh_depth = 0;
5879
5880         char* sql = buildSELECT( where_hash, query_hash, class_meta, ctx );
5881         if( !sql ) {
5882                 osrfLogDebug( OSRF_LOG_MARK, "Problem building query, returning NULL" );
5883                 *err = -1;
5884                 return NULL;
5885         }
5886
5887         osrfLogDebug( OSRF_LOG_MARK, "%s SQL =  %s", modulename, sql );
5888
5889         // Setting the timezone if requested and not in a transaction
5890         if (!getXactId(ctx)) {
5891                 if (tz) {
5892                         setenv("TZ",tz,1);
5893                         dbi_result tz_res = dbi_conn_queryf( writehandle, "SET timezone TO '%s'; -- cstore", tz );
5894                         if( !tz_res ) {
5895                                 osrfLogError( OSRF_LOG_MARK, "%s: Error setting timezone %s", modulename, tz);
5896                                 osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
5897                                         "osrfMethodException", ctx->request, "Error setting timezone" );
5898                                 if( !oilsIsDBConnected( writehandle )) {
5899                                         osrfAppSessionPanic( ctx->session );
5900                                         return -1;
5901                                 }
5902                         } else {
5903                                 dbi_result_free( tz_res );
5904                         }
5905                 } else {
5906                         unsetenv("TZ");
5907                         dbi_result res = dbi_conn_queryf( writehandle, "SET timezone TO DEFAULT; -- cstore" );
5908                         if( !res ) {
5909                                 osrfLogError( OSRF_LOG_MARK, "%s: Error resetting timezone", modulename);
5910                                 if( !oilsIsDBConnected( writehandle )) {
5911                                         osrfAppSessionPanic( ctx->session );
5912                                         return -1;
5913                                 }
5914                         } else {
5915                                 dbi_result_free( res );
5916                         }
5917                 }
5918         }
5919
5920
5921         dbi_result result = dbi_conn_query( dbhandle, sql );
5922
5923         if( NULL == result ) {
5924                 const char* msg;
5925                 int errnum = dbi_conn_error( dbhandle, &msg );
5926                 osrfLogError(OSRF_LOG_MARK, "%s: Error retrieving %s with query [%s]: %d %s",
5927                         modulename, osrfHashGet( class_meta, "fieldmapper" ), sql, errnum,
5928                         msg ? msg : "(No description available)" );
5929                 if( !oilsIsDBConnected( dbhandle ))
5930                         osrfAppSessionPanic( ctx->session );
5931                 osrfAppSessionStatus(
5932                         ctx->session,
5933                         OSRF_STATUS_INTERNALSERVERERROR,
5934                         "osrfMethodException",
5935                         ctx->request,
5936                         "Severe query error -- see error log for more details"
5937                 );
5938                 *err = -1;
5939                 free( sql );
5940                 return NULL;
5941
5942         } else {
5943                 osrfLogDebug( OSRF_LOG_MARK, "Query returned with no errors" );
5944
5945         }
5946
5947         jsonObject* res_list = jsonNewObjectType( JSON_ARRAY );
5948         jsonObject* row_obj = NULL;
5949
5950         // The following two steps are for verifyObjectPCRUD()'s benefit.
5951         // 1. get the flesh depth
5952         const jsonObject* _tmp = jsonObjectGetKeyConst( query_hash, "flesh" );
5953         if( _tmp ) {
5954                 flesh_depth = (int) jsonObjectGetNumber( _tmp );
5955                 if( flesh_depth == -1 || flesh_depth > max_flesh_depth )
5956                         flesh_depth = max_flesh_depth;
5957         }
5958
5959         // 2. figure out one consistent rs_size for verifyObjectPCRUD to use
5960         // over the whole life of this request.  This means if we've already set
5961         // up a rs_size_req_%d, do nothing.
5962         //      a. Incidentally, we can also use this opportunity to set i_respond_directly
5963         int *rs_size = osrfHashGetFmt( (osrfHash *) ctx->session->userData, "rs_size_req_%d", ctx->request );
5964         if( !rs_size ) {        // pointer null, so value not set in hash
5965                 // i_respond_directly can only be true at the /top/ of a recursive search, if even that.
5966                 i_respond_directly = ( *methodtype == 'r' || *methodtype == 'i' || *methodtype == 's' );
5967
5968                 rs_size = (int *) safe_malloc( sizeof(int) );   // will be freed by sessionDataFree()
5969                 unsigned long long result_count = dbi_result_get_numrows( result );
5970                 *rs_size = (int) result_count * (flesh_depth + 1);      // yes, we could lose some bits, but come on
5971                 osrfHashSet( (osrfHash *) ctx->session->userData, rs_size, "rs_size_req_%d", ctx->request );
5972         }
5973
5974         if( dbi_result_first_row( result )) {
5975
5976                 // Convert each row to a JSON_ARRAY of column values, and enclose those objects
5977                 // in a JSON_ARRAY of rows.  If two or more rows have the same key value, then
5978                 // eliminate the duplicates.
5979                 osrfLogDebug( OSRF_LOG_MARK, "Query returned at least one row" );
5980                 osrfHash* dedup = osrfNewHash();
5981                 do {
5982                         row_obj = oilsMakeFieldmapperFromResult( result, class_meta );
5983                         char* pkey_val = oilsFMGetString( row_obj, pkey );
5984                         if( osrfHashGet( dedup, pkey_val ) ) {
5985                                 jsonObjectFree( row_obj );
5986                                 free( pkey_val );
5987                         } else {
5988                                 if( !enforce_pcrud || !need_to_verify ||
5989                                                 verifyObjectPCRUD( ctx, class_meta, row_obj, 0 /* means check user data for rs_size */ )) {
5990                                         osrfHashSet( dedup, pkey_val, pkey_val );
5991                                         jsonObjectPush( res_list, row_obj );
5992                                 }
5993                         }
5994                 } while( dbi_result_next_row( result ));
5995                 osrfHashFree( dedup );
5996
5997         } else {
5998                 osrfLogDebug( OSRF_LOG_MARK, "%s returned no results for query %s",
5999                         modulename, sql );
6000         }
6001
6002         /* clean up the query */
6003         dbi_result_free( result );
6004         free( sql );
6005
6006         // If we're asked to flesh, and there's anything to flesh, then flesh it
6007         // (formerly we would skip fleshing if in pcrud mode, but now we support
6008         // fleshing even in PCRUD).
6009         if( res_list->size ) {
6010                 jsonObject* temp_blob;  // We need a non-zero flesh depth, and a list of fields to flesh
6011                 jsonObject* flesh_fields;
6012                 jsonObject* flesh_blob = NULL;
6013                 osrfStringArray* link_fields = NULL;
6014                 osrfHash* links = NULL;
6015                 int want_flesh = 0;
6016
6017                 if( query_hash ) {
6018                         temp_blob = jsonObjectGetKey( query_hash, "flesh_fields" );
6019                         if( temp_blob && flesh_depth > 0 ) {
6020
6021                                 flesh_blob = jsonObjectClone( temp_blob );
6022                                 flesh_fields = jsonObjectGetKey( flesh_blob, core_class );
6023
6024                                 links = osrfHashGet( class_meta, "links" );
6025
6026                                 // Make an osrfStringArray of the names of fields to be fleshed
6027                                 if( flesh_fields ) {
6028                                         if( flesh_fields->size == 1 ) {
6029                                                 const char* _t = jsonObjectGetString(
6030                                                         jsonObjectGetIndex( flesh_fields, 0 ) );
6031                                                 if( !strcmp( _t, "*" ))
6032                                                         link_fields = osrfHashKeys( links );
6033                                         }
6034
6035                                         if( !link_fields ) {
6036                                                 jsonObject* _f;
6037                                                 link_fields = osrfNewStringArray( 1 );
6038                                                 jsonIterator* _i = jsonNewIterator( flesh_fields );
6039                                                 while ((_f = jsonIteratorNext( _i ))) {
6040                                                         osrfStringArrayAdd( link_fields, jsonObjectGetString( _f ) );
6041                                                 }
6042                                                 jsonIteratorFree( _i );
6043                                         }
6044                                 }
6045                                 want_flesh = link_fields ? 1 : 0;
6046                         }
6047                 }
6048
6049                 osrfHash* fields = osrfHashGet( class_meta, "fields" );
6050
6051                 // Iterate over the JSON_ARRAY of rows
6052                 jsonObject* cur;
6053                 unsigned long res_idx = 0;
6054                 while((cur = jsonObjectGetIndex( res_list, res_idx++ ) )) {
6055
6056                         int i = 0;
6057                         const char* link_field;
6058
6059                         // Iterate over the list of fleshable fields
6060                         if ( want_flesh ) {
6061                                 while( (link_field = osrfStringArrayGetString(link_fields, i++)) ) {
6062
6063                                         osrfLogDebug( OSRF_LOG_MARK, "Starting to flesh %s", link_field );
6064
6065                                         osrfHash* kid_link = osrfHashGet( links, link_field );
6066                                         if( !kid_link )
6067                                                 continue;     // Not a link field; skip it
6068
6069                                         osrfHash* field = osrfHashGet( fields, link_field );
6070                                         if( !field )
6071                                                 continue;     // Not a field at all; skip it (IDL is ill-formed)
6072
6073                                         osrfHash* kid_idl = osrfHashGet( oilsIDL(),
6074                                                 osrfHashGet( kid_link, "class" ));
6075                                         if( !kid_idl )
6076                                                 continue;   // The class it links to doesn't exist; skip it
6077
6078                                         const char* reltype = osrfHashGet( kid_link, "reltype" );
6079                                         if( !reltype )
6080                                                 continue;   // No reltype; skip it (IDL is ill-formed)
6081
6082                                         osrfHash* value_field = field;
6083
6084                                         if(    !strcmp( reltype, "has_many" )
6085                                                 || !strcmp( reltype, "might_have" ) ) { // has_many or might_have
6086                                                 value_field = osrfHashGet(
6087                                                         fields, osrfHashGet( class_meta, "primarykey" ) );
6088                                         }
6089
6090                                         int kid_has_controller = osrfStringArrayContains( osrfHashGet(kid_idl, "controller"), modulename );
6091                                         // fleshing pcrud case: we require the controller in need_to_verify mode
6092                                         if ( !kid_has_controller && enforce_pcrud && need_to_verify ) {
6093                                                 osrfLogInfo( OSRF_LOG_MARK, "%s is not listed as a controller for %s; moving on", modulename, core_class );
6094
6095                                                 jsonObjectSetIndex(
6096                                                         cur,
6097                                                         (unsigned long) atoi( osrfHashGet(field, "array_position") ),
6098                                                         jsonNewObjectType(
6099                                                                 !strcmp( reltype, "has_many" ) ? JSON_ARRAY : JSON_NULL
6100                                                         )
6101                                                 );
6102                                                 continue;
6103                                         }
6104
6105                                         osrfStringArray* link_map = osrfHashGet( kid_link, "map" );
6106
6107                                         if( link_map->size > 0 ) {
6108                                                 jsonObject* _kid_key = jsonNewObjectType( JSON_ARRAY );
6109                                                 jsonObjectPush(
6110                                                         _kid_key,
6111                                                         jsonNewObject( osrfStringArrayGetString( link_map, 0 ) )
6112                                                 );
6113
6114                                                 jsonObjectSetKey(
6115                                                         flesh_blob,
6116                                                         osrfHashGet( kid_link, "class" ),
6117                                                         _kid_key
6118                                                 );
6119                                         };
6120
6121                                         osrfLogDebug(
6122                                                 OSRF_LOG_MARK,
6123                                                 "Link field: %s, remote class: %s, fkey: %s, reltype: %s",
6124                                                 osrfHashGet( kid_link, "field" ),
6125                                                 osrfHashGet( kid_link, "class" ),
6126                                                 osrfHashGet( kid_link, "key" ),
6127                                                 osrfHashGet( kid_link, "reltype" )
6128                                         );
6129
6130                                         const char* search_key = jsonObjectGetString(
6131                                                 jsonObjectGetIndex( cur,
6132                                                         atoi( osrfHashGet( value_field, "array_position" ) )
6133                                                 )
6134                                         );
6135
6136                                         if( !search_key ) {
6137                                                 osrfLogDebug( OSRF_LOG_MARK, "Nothing to search for!" );
6138                                                 continue;
6139                                         }
6140
6141                                         osrfLogDebug( OSRF_LOG_MARK, "Creating param objects..." );
6142
6143                                         // construct WHERE clause
6144                                         jsonObject* where_clause  = jsonNewObjectType( JSON_HASH );
6145                                         jsonObjectSetKey(
6146                                                 where_clause,
6147                                                 osrfHashGet( kid_link, "key" ),
6148                                                 jsonNewObject( search_key )
6149                                         );
6150
6151                                         // construct the rest of the query, mostly
6152                                         // by copying pieces of the previous level of query
6153                                         jsonObject* rest_of_query = jsonNewObjectType( JSON_HASH );
6154                                         jsonObjectSetKey( rest_of_query, "flesh",
6155                                                 jsonNewNumberObject( flesh_depth - 1 + link_map->size )
6156                                         );
6157
6158                                         if( flesh_blob )
6159                                                 jsonObjectSetKey( rest_of_query, "flesh_fields",
6160                                                         jsonObjectClone( flesh_blob ));
6161
6162                                         if( jsonObjectGetKeyConst( query_hash, "order_by" )) {
6163                                                 jsonObjectSetKey( rest_of_query, "order_by",
6164                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "order_by" ))
6165                                                 );
6166                                         }
6167
6168                                         if( jsonObjectGetKeyConst( query_hash, "select" )) {
6169                                                 jsonObjectSetKey( rest_of_query, "select",
6170                                                         jsonObjectClone( jsonObjectGetKeyConst( query_hash, "select" ))
6171                                                 );
6172                                         }
6173
6174                                         // do the query, recursively, to expand the fleshable field
6175                                         jsonObject* kids = doFieldmapperSearch( ctx, kid_idl,
6176                                                 where_clause, rest_of_query, err );
6177
6178                                         jsonObjectFree( where_clause );
6179                                         jsonObjectFree( rest_of_query );
6180
6181                                         if( *err ) {
6182                                                 osrfStringArrayFree( link_fields );
6183                                                 jsonObjectFree( res_list );
6184                                                 jsonObjectFree( flesh_blob );
6185                                                 return NULL;
6186                                         }
6187
6188                                         osrfLogDebug( OSRF_LOG_MARK, "Search for %s return %d linked objects",
6189                                                 osrfHashGet( kid_link, "class" ), kids->size );
6190
6191                                         // Traverse the result set
6192                                         jsonObject* X = NULL;
6193                                         if( link_map->size > 0 && kids->size > 0 ) {
6194                                                 X = kids;
6195                                                 kids = jsonNewObjectType( JSON_ARRAY );
6196
6197                                                 jsonObject* _k_node;
6198                                                 unsigned long res_idx = 0;
6199                                                 while((_k_node = jsonObjectGetIndex( X, res_idx++ ) )) {
6200                                                         jsonObjectPush(
6201                                                                 kids,
6202                                                                 jsonObjectClone(
6203                                                                         jsonObjectGetIndex(
6204                                                                                 _k_node,
6205                                                                                 (unsigned long) atoi(
6206                                                                                         osrfHashGet(
6207                                                                                                 osrfHashGet(
6208                                                                                                         osrfHashGet(
6209                                                                                                                 osrfHashGet(
6210                                                                                                                         oilsIDL(),
6211                                                                                                                         osrfHashGet( kid_link, "class" )
6212                                                                                                                 ),
6213                                                                                                                 "fields"
6214                                                                                                         ),
6215                                                                                                         osrfStringArrayGetString( link_map, 0 )
6216                                                                                                 ),
6217                                                                                                 "array_position"
6218                                                                                         )
6219                                                                                 )
6220                                                                         )
6221                                                                 )
6222                                                         );
6223                                                 } // end while loop traversing X
6224                                         }
6225
6226                                         if (kids->size > 0) {
6227
6228                                                 if((   !strcmp( osrfHashGet( kid_link, "reltype" ), "has_a" )
6229                                                         || !strcmp( osrfHashGet( kid_link, "reltype" ), "might_have" ))
6230                                                 ) {
6231                                                         osrfLogDebug(OSRF_LOG_MARK, "Storing fleshed objects in %s",
6232                                                                 osrfHashGet( kid_link, "field" ));
6233                                                         jsonObjectSetIndex(
6234                                                                 cur,
6235                                                                 (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6236                                                                 jsonObjectClone( jsonObjectGetIndex( kids, 0 ))
6237                                                         );
6238                                                 }
6239                                         }
6240
6241                                         if( !strcmp( osrfHashGet( kid_link, "reltype" ), "has_many" )) {
6242                                                 // has_many
6243                                                 osrfLogDebug( OSRF_LOG_MARK, "Storing fleshed objects in %s",
6244                                                         osrfHashGet( kid_link, "field" ) );
6245                                                 jsonObjectSetIndex(
6246                                                         cur,
6247                                                         (unsigned long) atoi( osrfHashGet( field, "array_position" ) ),
6248                                                         jsonObjectClone( kids )
6249                                                 );
6250                                         }
6251
6252                                         if( X ) {
6253                                                 jsonObjectFree( kids );
6254                                                 kids = X;
6255                                         }
6256
6257                                         jsonObjectFree( kids );
6258
6259                                         osrfLogDebug( OSRF_LOG_MARK, "Fleshing of %s complete",
6260                                                 osrfHashGet( kid_link, "field" ) );
6261                                         osrfLogDebug( OSRF_LOG_MARK, "%s", jsonObjectToJSON( cur ));
6262
6263                                 } // end while loop traversing list of fleshable fields
6264                         }
6265
6266                         if( i_respond_directly ) {
6267                                 if ( *methodtype == 'i' ) {
6268                                         osrfAppRespond( ctx,
6269                                                 oilsFMGetObject( cur, osrfHashGet( class_meta, "primarykey" ) ) );
6270                                 } else {
6271                                         osrfAppRespond( ctx, cur );
6272                                 }
6273                         }
6274                 } // end while loop traversing res_list
6275                 jsonObjectFree( flesh_blob );
6276                 osrfStringArrayFree( link_fields );
6277         }
6278
6279         if( i_respond_directly ) {
6280                 jsonObjectFree( res_list );
6281                 return jsonNewObjectType( JSON_ARRAY );
6282         } else {
6283                 return res_list;
6284         }
6285 }
6286
6287
6288 int doUpdate( osrfMethodContext* ctx ) {
6289         if( osrfMethodVerifyContext( ctx )) {
6290                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6291                 return -1;
6292         }
6293
6294         if( enforce_pcrud )
6295                 timeout_needs_resetting = 1;
6296
6297         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6298
6299         jsonObject* target = NULL;
6300         if( enforce_pcrud )
6301                 target = jsonObjectGetIndex( ctx->params, 1 );
6302         else
6303                 target = jsonObjectGetIndex( ctx->params, 0 );
6304
6305         if(!verifyObjectClass( ctx, target )) {
6306                 osrfAppRespondComplete( ctx, NULL );
6307                 return -1;
6308         }
6309
6310         if( getXactId( ctx ) == NULL ) {
6311                 osrfAppSessionStatus(
6312                         ctx->session,
6313                         OSRF_STATUS_BADREQUEST,
6314                         "osrfMethodException",
6315                         ctx->request,
6316                         "No active transaction -- required for UPDATE"
6317                 );
6318                 osrfAppRespondComplete( ctx, NULL );
6319                 return -1;
6320         }
6321
6322         // The following test is harmless but redundant.  If a class is
6323         // readonly, we don't register an update method for it.
6324         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6325                 osrfAppSessionStatus(
6326                         ctx->session,
6327                         OSRF_STATUS_BADREQUEST,
6328                         "osrfMethodException",
6329                         ctx->request,
6330                         "Cannot UPDATE readonly class"
6331                 );
6332                 osrfAppRespondComplete( ctx, NULL );
6333                 return -1;
6334         }
6335
6336         const char* trans_id = getXactId( ctx );
6337
6338         // Set the last_xact_id
6339         int index = oilsIDL_ntop( target->classname, "last_xact_id" );
6340         if( index > -1 ) {
6341                 osrfLogDebug( OSRF_LOG_MARK, "Setting last_xact_id to %s on %s at position %d",
6342                                 trans_id, target->classname, index );
6343                 jsonObjectSetIndex( target, index, jsonNewObject( trans_id ));
6344         }
6345
6346         char* pkey = osrfHashGet( meta, "primarykey" );
6347         osrfHash* fields = osrfHashGet( meta, "fields" );
6348
6349         char* id = oilsFMGetString( target, pkey );
6350
6351         osrfLogDebug(
6352                 OSRF_LOG_MARK,
6353                 "%s updating %s object with %s = %s",
6354                 modulename,
6355                 osrfHashGet( meta, "fieldmapper" ),
6356                 pkey,
6357                 id
6358         );
6359
6360         dbhandle = writehandle;
6361         growing_buffer* sql = buffer_init( 128 );
6362         buffer_fadd( sql,"UPDATE %s SET", osrfHashGet( meta, "tablename" ));
6363
6364         int first = 1;
6365         osrfHash* field_def = NULL;
6366         osrfHashIterator* field_itr = osrfNewHashIterator( fields );
6367         while( ( field_def = osrfHashIteratorNext( field_itr ) ) ) {
6368
6369                 // Skip virtual fields, and the primary key
6370                 if( str_is_true( osrfHashGet( field_def, "virtual") ) )
6371                         continue;
6372
6373                 if (osrfStringArrayContains( osrfHashGet(field_def, "suppress_controller"), modulename ))
6374                         continue;
6375
6376
6377                 const char* field_name = osrfHashIteratorKey( field_itr );
6378                 if( ! strcmp( field_name, pkey ) )
6379                         continue;
6380
6381                 const jsonObject* field_object = oilsFMGetObject( target, field_name );
6382
6383                 int value_is_numeric = 0;    // boolean
6384                 char* value;
6385                 if( field_object && field_object->classname ) {
6386                         value = oilsFMGetString(
6387                                 field_object,
6388                                 (char*) oilsIDLFindPath( "/%s/primarykey", field_object->classname )
6389                         );
6390                 } else if( field_object && JSON_BOOL == field_object->type ) {
6391                         if( jsonBoolIsTrue( field_object ) )
6392                                 value = strdup( "t" );
6393                         else
6394                                 value = strdup( "f" );
6395                 } else {
6396                         value = jsonObjectToSimpleString( field_object );
6397                         if( field_object && JSON_NUMBER == field_object->type )
6398                                 value_is_numeric = 1;
6399                 }
6400
6401                 osrfLogDebug( OSRF_LOG_MARK, "Updating %s object with %s = %s",
6402                                 osrfHashGet( meta, "fieldmapper" ), field_name, value);
6403
6404                 if( !field_object || field_object->type == JSON_NULL ) {
6405                         if( !( !( strcmp( osrfHashGet( meta, "classname" ), "au" ) )
6406                                         && !( strcmp( field_name, "passwd" ) )) ) { // arg at the special case!
6407                                 if( first )
6408                                         first = 0;
6409                                 else
6410                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6411                                 buffer_fadd( sql, " %s = NULL", field_name );
6412                         }
6413
6414                 } else if( value_is_numeric || !strcmp( get_primitive( field_def ), "number") ) {
6415                         if( first )
6416                                 first = 0;
6417                         else
6418                                 OSRF_BUFFER_ADD_CHAR( sql, ',' );
6419
6420                         const char* numtype = get_datatype( field_def );
6421                         if( !strncmp( numtype, "INT", 3 ) ) {
6422                                 buffer_fadd( sql, " %s = %ld", field_name, atol( value ) );
6423                         } else if( !strcmp( numtype, "NUMERIC" ) ) {
6424                                 buffer_fadd( sql, " %s = %f", field_name, atof( value ) );
6425                         } else {
6426                                 // Must really be intended as a string, so quote it
6427                                 if( dbi_conn_quote_string( dbhandle, &value )) {
6428                                         buffer_fadd( sql, " %s = %s", field_name, value );
6429                                 } else {
6430                                         osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]",
6431                                                 modulename, value );
6432                                         osrfAppSessionStatus(
6433                                                 ctx->session,
6434                                                 OSRF_STATUS_INTERNALSERVERERROR,
6435                                                 "osrfMethodException",
6436                                                 ctx->request,
6437                                                 "Error quoting string -- please see the error log for more details"
6438                                         );
6439                                         free( value );
6440                                         free( id );
6441                                         osrfHashIteratorFree( field_itr );
6442                                         buffer_free( sql );
6443                                         osrfAppRespondComplete( ctx, NULL );
6444                                         return -1;
6445                                 }
6446                         }
6447
6448                         osrfLogDebug( OSRF_LOG_MARK, "%s is of type %s", field_name, numtype );
6449
6450                 } else {
6451                         if( dbi_conn_quote_string( dbhandle, &value ) ) {
6452                                 if( first )
6453                                         first = 0;
6454                                 else
6455                                         OSRF_BUFFER_ADD_CHAR( sql, ',' );
6456                                 buffer_fadd( sql, " %s = %s", field_name, value );
6457                         } else {
6458                                 osrfLogError( OSRF_LOG_MARK, "%s: Error quoting string [%s]", modulename, value );
6459                                 osrfAppSessionStatus(
6460                                         ctx->session,
6461                                         OSRF_STATUS_INTERNALSERVERERROR,
6462                                         "osrfMethodException",
6463                                         ctx->request,
6464                                         "Error quoting string -- please see the error log for more details"
6465                                 );
6466                                 free( value );
6467                                 free( id );
6468                                 osrfHashIteratorFree( field_itr );
6469                                 buffer_free( sql );
6470                                 osrfAppRespondComplete( ctx, NULL );
6471                                 return -1;
6472                         }
6473                 }
6474
6475                 free( value );
6476
6477         } // end while
6478
6479         osrfHashIteratorFree( field_itr );
6480
6481         jsonObject* obj = jsonNewObject( id );
6482
6483         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey )), "number" ))
6484                 dbi_conn_quote_string( dbhandle, &id );
6485
6486         buffer_fadd( sql, " WHERE %s = %s;", pkey, id );
6487
6488         char* query = buffer_release( sql );
6489         osrfLogDebug( OSRF_LOG_MARK, "%s: Update SQL [%s]", modulename, query );
6490
6491         dbi_result result = dbi_conn_query( dbhandle, query );
6492         free( query );
6493
6494         int rc = 0;
6495         if( !result ) {
6496                 jsonObjectFree( obj );
6497                 obj = jsonNewObject( NULL );
6498                 const char* msg;
6499                 int errnum = dbi_conn_error( dbhandle, &msg );
6500                 osrfLogError(
6501                         OSRF_LOG_MARK,
6502                         "%s ERROR updating %s object with %s = %s: %d %s",
6503                         modulename,
6504                         osrfHashGet( meta, "fieldmapper" ),
6505                         pkey,
6506                         id,
6507                         errnum,
6508                         msg ? msg : "(No description available)"
6509                 );
6510                 osrfAppSessionStatus(
6511                         ctx->session,
6512                         OSRF_STATUS_INTERNALSERVERERROR,
6513                         "osrfMethodException",
6514                         ctx->request,
6515                         "Error in updating a row -- please see the error log for more details"
6516                 );
6517                 if( !oilsIsDBConnected( dbhandle ))
6518                         osrfAppSessionPanic( ctx->session );
6519                 rc = -1;
6520         } else
6521                 dbi_result_free( result );
6522
6523         free( id );
6524         osrfAppRespondComplete( ctx, obj );
6525         jsonObjectFree( obj );
6526         return rc;
6527 }
6528
6529 int doDelete( osrfMethodContext* ctx ) {
6530         if( osrfMethodVerifyContext( ctx )) {
6531                 osrfLogError( OSRF_LOG_MARK, "Invalid method context" );
6532                 return -1;
6533         }
6534
6535         if( enforce_pcrud )
6536                 timeout_needs_resetting = 1;
6537
6538         osrfHash* meta = osrfHashGet( (osrfHash*) ctx->method->userData, "class" );
6539
6540         if( getXactId( ctx ) == NULL ) {
6541                 osrfAppSessionStatus(
6542                         ctx->session,
6543                         OSRF_STATUS_BADREQUEST,
6544                         "osrfMethodException",
6545                         ctx->request,
6546                         "No active transaction -- required for DELETE"
6547                 );
6548                 osrfAppRespondComplete( ctx, NULL );
6549                 return -1;
6550         }
6551
6552         // The following test is harmless but redundant.  If a class is
6553         // readonly, we don't register a delete method for it.
6554         if( str_is_true( osrfHashGet( meta, "readonly" ) ) ) {
6555                 osrfAppSessionStatus(
6556                         ctx->session,
6557                         OSRF_STATUS_BADREQUEST,
6558                         "osrfMethodException",
6559                         ctx->request,
6560                         "Cannot DELETE readonly class"
6561                 );
6562                 osrfAppRespondComplete( ctx, NULL );
6563                 return -1;
6564         }
6565
6566         dbhandle = writehandle;
6567
6568         char* pkey = osrfHashGet( meta, "primarykey" );
6569
6570         int _obj_pos = 0;
6571         if( enforce_pcrud )
6572                 _obj_pos = 1;
6573
6574         char* id;
6575         if( jsonObjectGetIndex( ctx->params, _obj_pos )->classname ) {
6576                 if( !verifyObjectClass( ctx, jsonObjectGetIndex( ctx->params, _obj_pos ))) {
6577                         osrfAppRespondComplete( ctx, NULL );
6578                         return -1;
6579                 }
6580
6581                 id = oilsFMGetString( jsonObjectGetIndex(ctx->params, _obj_pos), pkey );
6582         } else {
6583                 if( enforce_pcrud && !verifyObjectPCRUD( ctx, meta, NULL, 1 )) {
6584                         osrfAppRespondComplete( ctx, NULL );
6585                         return -1;
6586                 }
6587                 id = jsonObjectToSimpleString( jsonObjectGetIndex( ctx->params, _obj_pos ));
6588         }
6589
6590         osrfLogDebug(
6591                 OSRF_LOG_MARK,
6592                 "%s deleting %s object with %s = %s",
6593                 modulename,
6594                 osrfHashGet( meta, "fieldmapper" ),
6595                 pkey,
6596                 id
6597         );
6598
6599         jsonObject* obj = jsonNewObject( id );
6600
6601         if( strcmp( get_primitive( osrfHashGet( osrfHashGet(meta, "fields"), pkey ) ), "number" ) )
6602                 dbi_conn_quote_string( writehandle, &id );
6603
6604         dbi_result result = dbi_conn_queryf( writehandle, "DELETE FROM %s WHERE %s = %s;",
6605                 osrfHashGet( meta, "tablename" ), pkey, id );
6606
6607         int rc = 0;
6608         if( !result ) {
6609                 rc = -1;
6610                 jsonObjectFree( obj );
6611                 obj = jsonNewObject( NULL );
6612                 const char* msg;
6613                 int errnum = dbi_conn_error( writehandle, &msg );
6614                 osrfLogError(
6615                         OSRF_LOG_MARK,
6616                         "%s ERROR deleting %s object with %s = %s: %d %s",
6617                         modulename,
6618                         osrfHashGet( meta, "fieldmapper" ),
6619                         pkey,
6620                         id,
6621                         errnum,
6622                         msg ? msg : "(No description available)"
6623                 );
6624                 osrfAppSessionStatus(
6625                         ctx->session,
6626                         OSRF_STATUS_INTERNALSERVERERROR,
6627                         "osrfMethodException",
6628                         ctx->request,
6629                         "Error in deleting a row -- please see the error log for more details"
6630                 );
6631                 if( !oilsIsDBConnected( writehandle ))
6632                         osrfAppSessionPanic( ctx->session );
6633         } else
6634                 dbi_result_free( result );
6635
6636         free( id );
6637
6638         osrfAppRespondComplete( ctx, obj );
6639         jsonObjectFree( obj );
6640         return rc;
6641 }
6642
6643 /**
6644         @brief Translate a row returned from the database into a jsonObject of type JSON_ARRAY.
6645         @param result An iterator for a result set; we only look at the current row.
6646         @param @meta Pointer to the class metadata for the core class.
6647         @return Pointer to the resulting jsonObject if successful; otherwise NULL.
6648
6649         If a column is not defined in the IDL, or if it has no array_position defined for it in
6650         the IDL, or if it is defined as virtual, ignore it.
6651
6652         Otherwise, translate the column value into a jsonObject of type JSON_NULL, JSON_NUMBER,
6653         or JSON_STRING.  Then insert this jsonObject into the JSON_ARRAY according to its
6654         array_position in the IDL.
6655
6656         A field defined in the IDL but not represented in the returned row will leave a hole
6657         in the JSON_ARRAY.  In effect it will be treated as a null value.
6658
6659         In the resulting JSON_ARRAY, the field values appear in the sequence defined by the IDL,
6660         regardless of their sequence in the SELECT statement.  The JSON_ARRAY is assigned the
6661         classname corresponding to the @a meta argument.
6662
6663         The calling code is responsible for freeing the the resulting jsonObject by calling
6664         jsonObjectFree().
6665 */
6666 static jsonObject* oilsMakeFieldmapperFromResult( dbi_result result, osrfHash* meta) {
6667         if( !( result && meta )) return NULL;
6668
6669         jsonObject* object = jsonNewObjectType( JSON_ARRAY );
6670         jsonObjectSetClass( object, osrfHashGet( meta, "classname" ));
6671         osrfLogInternal( OSRF_LOG_MARK, "Setting object class to %s ", object->classname );
6672
6673         osrfHash* fields = osrfHashGet( meta, "fields" );
6674
6675         int columnIndex = 1;
6676         const char* columnName;
6677
6678         /* cycle through the columns in the row returned from the database */
6679         while( (columnName = dbi_result_get_field_name( result, columnIndex )) ) {
6680
6681                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6682
6683                 int fmIndex = -1;  // Will be set to the IDL's sequence number for this field
6684
6685                 /* determine the field type and storage attributes */
6686                 unsigned short type = dbi_result_get_field_type_idx( result, columnIndex );
6687                 int attr            = dbi_result_get_field_attribs_idx( result, columnIndex );
6688
6689                 // Fetch the IDL's sequence number for the field.  If the field isn't in the IDL,
6690                 // or if it has no sequence number there, or if it's virtual, skip it.
6691                 osrfHash* _f = osrfHashGet( fields, (char*) columnName );
6692                 if( _f ) {
6693
6694                         if( str_is_true( osrfHashGet( _f, "virtual" )))
6695                                 continue;   // skip this column: IDL says it's virtual
6696
6697                         const char* pos = (char*) osrfHashGet( _f, "array_position" );
6698                         if( !pos )      // IDL has no sequence number for it.  This shouldn't happen,
6699                                 continue;    // since we assign sequence numbers dynamically as we load the IDL.
6700
6701                         fmIndex = atoi( pos );
6702                         osrfLogInternal( OSRF_LOG_MARK, "... Found column at position [%s]...", pos );
6703                 } else {
6704                         continue;     // This field is not defined in the IDL
6705                 }
6706
6707                 // Stuff the column value into a slot in the JSON_ARRAY, indexed according to the
6708                 // sequence number from the IDL (which is likely to be different from the sequence
6709                 // of columns in the SELECT clause).
6710                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6711                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( NULL ));
6712                 } else {
6713
6714                         switch( type ) {
6715
6716                                 case DBI_TYPE_INTEGER :
6717
6718                                         if( attr & DBI_INTEGER_SIZE8 )
6719                                                 jsonObjectSetIndex( object, fmIndex,
6720                                                         jsonNewNumberObject(
6721                                                                 dbi_result_get_longlong_idx( result, columnIndex )));
6722                                         else
6723                                                 jsonObjectSetIndex( object, fmIndex,
6724                                                         jsonNewNumberObject( dbi_result_get_int_idx( result, columnIndex )));
6725
6726                                         break;
6727
6728                                 case DBI_TYPE_DECIMAL :
6729                                         jsonObjectSetIndex( object, fmIndex,
6730                                                         jsonNewNumberObject( dbi_result_get_double_idx(result, columnIndex )));
6731                                         break;
6732
6733                                 case DBI_TYPE_STRING :
6734
6735                                         jsonObjectSetIndex(
6736                                                 object,
6737                                                 fmIndex,
6738                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex ))
6739                                         );
6740
6741                                         break;
6742
6743                                 case DBI_TYPE_DATETIME : {
6744
6745                                         char dt_string[ 256 ] = "";
6746                                         struct tm gmdt;
6747
6748                                         // Fetch the date column as a time_t
6749                                         time_t _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6750
6751                                         // Translate the time_t to a human-readable string
6752                                         if( !( attr & DBI_DATETIME_DATE )) {
6753                                                 gmtime_r( &_tmp_dt, &gmdt );
6754                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6755                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6756                                                 gmtime_r( &_tmp_dt, &gmdt );
6757                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6758                                         } else {
6759                                                 localtime_r( &_tmp_dt, &gmdt );
6760                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6761                                         }
6762
6763                                         jsonObjectSetIndex( object, fmIndex, jsonNewObject( dt_string ));
6764
6765                                         break;
6766                                 }
6767                                 case DBI_TYPE_BINARY :
6768                                         osrfLogError( OSRF_LOG_MARK,
6769                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6770                         } // End switch
6771                 }
6772                 ++columnIndex;
6773         } // End while
6774
6775         return object;
6776 }
6777
6778 static jsonObject* oilsMakeJSONFromResult( dbi_result result ) {
6779         if( !result ) return NULL;
6780
6781         jsonObject* object = jsonNewObject( NULL );
6782
6783         time_t _tmp_dt;
6784         char dt_string[ 256 ];
6785         struct tm gmdt;
6786
6787         int fmIndex;
6788         int columnIndex = 1;
6789         int attr;
6790         unsigned short type;
6791         const char* columnName;
6792
6793         /* cycle through the column list */
6794         while(( columnName = dbi_result_get_field_name( result, columnIndex ))) {
6795
6796                 osrfLogInternal( OSRF_LOG_MARK, "Looking for column named [%s]...", (char*) columnName );
6797
6798                 fmIndex = -1; // reset the position
6799
6800                 /* determine the field type and storage attributes */
6801                 type = dbi_result_get_field_type_idx( result, columnIndex );
6802                 attr = dbi_result_get_field_attribs_idx( result, columnIndex );
6803
6804                 if( dbi_result_field_is_null_idx( result, columnIndex )) {
6805                         jsonObjectSetKey( object, columnName, jsonNewObject( NULL ));
6806                 } else {
6807
6808                         switch( type ) {
6809
6810                                 case DBI_TYPE_INTEGER :
6811
6812                                         if( attr & DBI_INTEGER_SIZE8 )
6813                                                 jsonObjectSetKey( object, columnName,
6814                                                                 jsonNewNumberObject( dbi_result_get_longlong_idx(
6815                                                                                 result, columnIndex )) );
6816                                         else
6817                                                 jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6818                                                                 dbi_result_get_int_idx( result, columnIndex )) );
6819                                         break;
6820
6821                                 case DBI_TYPE_DECIMAL :
6822                                         jsonObjectSetKey( object, columnName, jsonNewNumberObject(
6823                                                 dbi_result_get_double_idx( result, columnIndex )) );
6824                                         break;
6825
6826                                 case DBI_TYPE_STRING :
6827                                         jsonObjectSetKey( object, columnName,
6828                                                 jsonNewObject( dbi_result_get_string_idx( result, columnIndex )));
6829                                         break;
6830
6831                                 case DBI_TYPE_DATETIME :
6832
6833                                         memset( dt_string, '\0', sizeof( dt_string ));
6834                                         memset( &gmdt, '\0', sizeof( gmdt ));
6835
6836                                         _tmp_dt = dbi_result_get_datetime_idx( result, columnIndex );
6837
6838                                         if( !( attr & DBI_DATETIME_DATE )) {
6839                                                 gmtime_r( &_tmp_dt, &gmdt );
6840                                                 strftime( dt_string, sizeof( dt_string ), "%T", &gmdt );
6841                                         } else if( !( attr & DBI_DATETIME_TIME )) {
6842                                                 gmtime_r( &_tmp_dt, &gmdt );
6843                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%d", &gmdt );
6844                                         } else {
6845                                                 localtime_r( &_tmp_dt, &gmdt );
6846                                                 strftime( dt_string, sizeof( dt_string ), "%04Y-%m-%dT%T%z", &gmdt );
6847                                         }
6848
6849                                         jsonObjectSetKey( object, columnName, jsonNewObject( dt_string ));
6850                                         break;
6851
6852                                 case DBI_TYPE_BINARY :
6853                                         osrfLogError( OSRF_LOG_MARK,
6854                                                 "Can't do binary at column %s : index %d", columnName, columnIndex );
6855                         }
6856                 }
6857                 ++columnIndex;
6858         } // end while loop traversing result
6859
6860         return object;
6861 }
6862
6863 // Interpret a string as true or false
6864 int str_is_true( const char* str ) {
6865         if( NULL == str || strcasecmp( str, "true" ) )
6866                 return 0;
6867         else
6868                 return 1;
6869 }
6870
6871 // Interpret a jsonObject as true or false
6872 static int obj_is_true( const jsonObject* obj ) {
6873         if( !obj )
6874                 return 0;
6875         else switch( obj->type )
6876         {
6877                 case JSON_BOOL :
6878                         if( obj->value.b )
6879                                 return 1;
6880                         else
6881                                 return 0;
6882                 case JSON_STRING :
6883                         if( strcasecmp( obj->value.s, "true" ) )
6884                                 return 0;
6885                         else
6886                                 return 1;
6887                 case JSON_NUMBER :          // Support 1/0 for perl's sake
6888                         if( jsonObjectGetNumber( obj ) == 1.0 )
6889                                 return 1;
6890                         else
6891                                 return 0;
6892                 default :
6893                         return 0;
6894         }
6895 }
6896
6897 // Translate a numeric code into a text string identifying a type of
6898 // jsonObject.  To be used for building error messages.
6899 static const char* json_type( int code ) {
6900         switch ( code )
6901         {
6902                 case 0 :
6903                         return "JSON_HASH";
6904                 case 1 :
6905                         return "JSON_ARRAY";
6906                 case 2 :
6907                         return "JSON_STRING";
6908                 case 3 :
6909                         return "JSON_NUMBER";
6910                 case 4 :
6911                         return "JSON_NULL";
6912                 case 5 :
6913                         return "JSON_BOOL";
6914                 default :
6915                         return "(unrecognized)";
6916         }
6917 }
6918
6919 // Extract the "primitive" attribute from an IDL field definition.
6920 // If we haven't initialized the app, then we must be running in
6921 // some kind of testbed.  In that case, default to "string".
6922 static const char* get_primitive( osrfHash* field ) {
6923         const char* s = osrfHashGet( field, "primitive" );
6924         if( !s ) {
6925                 if( child_initialized )
6926                         osrfLogError(
6927                                 OSRF_LOG_MARK,
6928                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
6929                                 modulename,
6930                                 osrfHashGet( field, "name" )
6931                         );
6932
6933                 s = "string";
6934         }
6935         return s;
6936 }
6937
6938 // Extract the "datatype" attribute from an IDL field definition.
6939 // If we haven't initialized the app, then we must be running in
6940 // some kind of testbed.  In that case, default to to NUMERIC,
6941 // since we look at the datatype only for numbers.
6942 static const char* get_datatype( osrfHash* field ) {
6943         const char* s = osrfHashGet( field, "datatype" );
6944         if( !s ) {
6945                 if( child_initialized )
6946                         osrfLogError(
6947                                 OSRF_LOG_MARK,
6948                                 "%s ERROR No \"datatype\" attribute for field \"%s\"",
6949                                 modulename,
6950                                 osrfHashGet( field, "name" )
6951                         );
6952                 else
6953                         s = "NUMERIC";
6954         }
6955         return s;
6956 }
6957
6958 /**
6959         @brief Determine whether a string is potentially a valid SQL identifier.
6960         @param s The identifier to be tested.
6961         @return 1 if the input string is potentially a valid SQL identifier, or 0 if not.
6962
6963         Purpose: to prevent certain kinds of SQL injection.  To that end we don't necessarily
6964         need to follow all the rules exactly, such as requiring that the first character not
6965         be a digit.
6966
6967         We allow leading and trailing white space.  In between, we do not allow punctuation
6968         (except for underscores and dollar signs), control characters, or embedded white space.
6969
6970         More pedantically we should allow quoted identifiers containing arbitrary characters, but
6971         for the foreseeable future such quoted identifiers are not likely to be an issue.
6972 */
6973 int is_identifier( const char* s) {
6974         if( !s )
6975                 return 0;
6976
6977         // Skip leading white space
6978         while( isspace( (unsigned char) *s ) )
6979                 ++s;
6980
6981         if( !s )
6982                 return 0;   // Nothing but white space?  Not okay.
6983
6984         // Check each character until we reach white space or
6985         // end-of-string.  Letters, digits, underscores, and
6986         // dollar signs are okay. With the exception of periods
6987         // (as in schema.identifier), control characters and other
6988         // punctuation characters are not okay.  Anything else
6989         // is okay -- it could for example be part of a multibyte
6990         // UTF8 character such as a letter with diacritical marks,
6991         // and those are allowed.
6992         do {
6993                 if( isalnum( (unsigned char) *s )
6994                         || '.' == *s
6995                         || '_' == *s
6996                         || '$' == *s )
6997                         ;  // Fine; keep going
6998                 else if(   ispunct( (unsigned char) *s )
6999                                 || iscntrl( (unsigned char) *s ) )
7000                         return 0;
7001                         ++s;
7002         } while( *s && ! isspace( (unsigned char) *s ) );
7003
7004         // If we found any white space in the above loop,
7005         // the rest had better be all white space.
7006
7007         while( isspace( (unsigned char) *s ) )
7008                 ++s;
7009
7010         if( *s )
7011                 return 0;   // White space was embedded within non-white space
7012
7013         return 1;
7014 }
7015
7016 /**
7017         @brief Determine whether to accept a character string as a comparison operator.
7018         @param op The candidate comparison operator.
7019         @return 1 if the string is acceptable as a comparison operator, or 0 if not.
7020
7021         We don't validate the operator for real.  We just make sure that it doesn't contain
7022         any semicolons or white space (with special exceptions for a few specific operators).
7023         The idea is to block certain kinds of SQL injection.  If it has no semicolons or white
7024         space but it's still not a valid operator, then the database will complain.
7025
7026         Another approach would be to compare the string against a short list of approved operators.
7027         We don't do that because we want to allow custom operators like ">100*", which at this
7028         writing would be difficult or impossible to express otherwise in a JSON query.
7029 */
7030 int is_good_operator( const char* op ) {
7031         if( !op ) return 0;   // Sanity check
7032
7033         const char* s = op;
7034         while( *s ) {
7035                 if( isspace( (unsigned char) *s ) ) {
7036                         // Special exceptions for SIMILAR TO, IS DISTINCT FROM,
7037                         // and IS NOT DISTINCT FROM.
7038                         if( !strcasecmp( op, "similar to" ) )
7039                                 return 1;
7040                         else if( !strcasecmp( op, "is distinct from" ) )
7041                                 return 1;
7042                         else if( !strcasecmp( op, "is not distinct from" ) )
7043                                 return 1;
7044                         else
7045                                 return 0;
7046                 }
7047                 else if( ';' == *s )
7048                         return 0;
7049                 ++s;
7050         }
7051         return 1;
7052 }
7053
7054 /**
7055         @name Query Frame Management
7056
7057         The following machinery supports a stack of query frames for use by SELECT().
7058
7059         A query frame caches information about one level of a SELECT query.  When we enter
7060         a subquery, we push another query frame onto the stack, and pop it off when we leave.
7061
7062         The query frame stores information about the core class, and about any joined classes
7063         in the FROM clause.
7064
7065         The main purpose is to map table aliases to classes and tables, so that a query can
7066         join to the same table more than once.  A secondary goal is to reduce the number of
7067         lookups in the IDL by caching the results.
7068 */
7069 /*@{*/
7070
7071 #define STATIC_CLASS_INFO_COUNT 3
7072
7073 static ClassInfo static_class_info[ STATIC_CLASS_INFO_COUNT ];
7074
7075 /**
7076         @brief Allocate a ClassInfo as raw memory.
7077         @return Pointer to the newly allocated ClassInfo.
7078
7079         Except for the in_use flag, which is used only by the allocation and deallocation
7080         logic, we don't initialize the ClassInfo here.
7081 */
7082 static ClassInfo* allocate_class_info( void ) {
7083         // In order to reduce the number of mallocs and frees, we return a static
7084         // instance of ClassInfo, if we can find one that we're not already using.
7085         // We rely on the fact that the compiler will implicitly initialize the
7086         // static instances so that in_use == 0.
7087
7088         int i;
7089         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7090                 if( ! static_class_info[ i ].in_use ) {
7091                         static_class_info[ i ].in_use = 1;
7092                         return static_class_info + i;
7093                 }
7094         }
7095
7096         // The static ones are all in use.  Malloc one.
7097
7098         return safe_malloc( sizeof( ClassInfo ) );
7099 }
7100
7101 /**
7102         @brief Free any malloc'd memory owned by a ClassInfo, returning it to a pristine state.
7103         @param info Pointer to the ClassInfo to be cleared.
7104 */
7105 static void clear_class_info( ClassInfo* info ) {
7106         // Sanity check
7107         if( ! info )
7108                 return;
7109
7110         // Free any malloc'd strings
7111
7112         if( info->alias != info->alias_store )
7113                 free( info->alias );
7114
7115         if( info->class_name != info->class_name_store )
7116                 free( info->class_name );
7117
7118         free( info->source_def );
7119
7120         info->alias = info->class_name = info->source_def = NULL;
7121         info->next = NULL;
7122 }
7123
7124 /**
7125         @brief Free a ClassInfo and everything it owns.
7126         @param info Pointer to the ClassInfo to be freed.
7127 */
7128 static void free_class_info( ClassInfo* info ) {
7129         // Sanity check
7130         if( ! info )
7131                 return;
7132
7133         clear_class_info( info );
7134
7135         // If it's one of the static instances, just mark it as not in use
7136
7137         int i;
7138         for( i = 0; i < STATIC_CLASS_INFO_COUNT; ++i ) {
7139                 if( info == static_class_info + i ) {
7140                         static_class_info[ i ].in_use = 0;
7141                         return;
7142                 }
7143         }
7144
7145         // Otherwise it must have been malloc'd, so free it
7146
7147         free( info );
7148 }
7149
7150 /**
7151         @brief Populate an already-allocated ClassInfo.
7152         @param info Pointer to the ClassInfo to be populated.
7153         @param alias Alias for the class.  If it is NULL, or an empty string, use the class
7154         name for an alias.
7155         @param class Name of the class.
7156         @return Zero if successful, or 1 if not.
7157
7158         Populate the ClassInfo with copies of the alias and class name, and with pointers to
7159         the relevant portions of the IDL for the specified class.
7160 */
7161 static int build_class_info( ClassInfo* info, const char* alias, const char* class ) {
7162         // Sanity checks
7163         if( ! info ){
7164                 osrfLogError( OSRF_LOG_MARK,
7165                                           "%s ERROR: No ClassInfo available to populate", modulename );
7166                 info->alias = info->class_name = info->source_def = NULL;
7167                 info->class_def = info->fields = info->links = NULL;
7168                 return 1;
7169         }
7170
7171         if( ! class ) {
7172                 osrfLogError( OSRF_LOG_MARK,
7173                                           "%s ERROR: No class name provided for lookup", modulename );
7174                 info->alias = info->class_name = info->source_def = NULL;
7175                 info->class_def = info->fields = info->links = NULL;
7176                 return 1;
7177         }
7178
7179         // Alias defaults to class name if not supplied
7180         if( ! alias || ! alias[ 0 ] )
7181                 alias = class;
7182
7183         // Look up class info in the IDL
7184         osrfHash* class_def = osrfHashGet( oilsIDL(), class );
7185         if( ! class_def ) {
7186                 osrfLogError( OSRF_LOG_MARK,
7187                                           "%s ERROR: Class %s not defined in IDL", modulename, class );
7188                 info->alias = info->class_name = info->source_def = NULL;
7189                 info->class_def = info->fields = info->links = NULL;
7190                 return 1;
7191         } else if( str_is_true( osrfHashGet( class_def, "virtual" ) ) ) {
7192                 osrfLogError( OSRF_LOG_MARK,
7193                                           "%s ERROR: Class %s is defined as virtual", modulename, class );
7194                 info->alias = info->class_name = info->source_def = NULL;
7195                 info->class_def = info->fields = info->links = NULL;
7196                 return 1;
7197         }
7198
7199         osrfHash* links = osrfHashGet( class_def, "links" );
7200         if( ! links ) {
7201                 osrfLogError( OSRF_LOG_MARK,
7202                                           "%s ERROR: No links defined in IDL for class %s", modulename, class );
7203                 info->alias = info->class_name = info->source_def = NULL;
7204                 info->class_def = info->fields = info->links = NULL;
7205                 return 1;
7206         }
7207
7208         osrfHash* fields = osrfHashGet( class_def, "fields" );
7209         if( ! fields ) {
7210                 osrfLogError( OSRF_LOG_MARK,
7211                                           "%s ERROR: No fields defined in IDL for class %s", modulename, class );
7212                 info->alias = info->class_name = info->source_def = NULL;
7213                 info->class_def = info->fields = info->links = NULL;
7214                 return 1;
7215         }
7216
7217         char* source_def = oilsGetRelation( class_def );
7218         if( ! source_def )
7219                 return 1;
7220
7221         // We got everything we need, so populate the ClassInfo
7222         if( strlen( alias ) > ALIAS_STORE_SIZE )
7223                 info->alias = strdup( alias );
7224         else {
7225                 strcpy( info->alias_store, alias );
7226                 info->alias = info->alias_store;
7227         }
7228
7229         if( strlen( class ) > CLASS_NAME_STORE_SIZE )
7230                 info->class_name = strdup( class );
7231         else {
7232                 strcpy( info->class_name_store, class );
7233                 info->class_name = info->class_name_store;
7234         }
7235
7236         info->source_def = source_def;
7237
7238         info->class_def = class_def;
7239         info->links     = links;
7240         info->fields    = fields;
7241
7242         return 0;
7243 }
7244
7245 #define STATIC_FRAME_COUNT 3
7246
7247 static QueryFrame static_frame[ STATIC_FRAME_COUNT ];
7248
7249 /**
7250         @brief Allocate a QueryFrame as raw memory.
7251         @return Pointer to the newly allocated QueryFrame.
7252
7253         Except for the in_use flag, which is used only by the allocation and deallocation
7254         logic, we don't initialize the QueryFrame here.
7255 */
7256 static QueryFrame* allocate_frame( void ) {
7257         // In order to reduce the number of mallocs and frees, we return a static
7258         // instance of QueryFrame, if we can find one that we're not already using.
7259         // We rely on the fact that the compiler will implicitly initialize the
7260         // static instances so that in_use == 0.
7261
7262         int i;
7263         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7264                 if( ! static_frame[ i ].in_use ) {
7265                         static_frame[ i ].in_use = 1;
7266                         return static_frame + i;
7267                 }
7268         }
7269
7270         // The static ones are all in use.  Malloc one.
7271
7272         return safe_malloc( sizeof( QueryFrame ) );
7273 }
7274
7275 /**
7276         @brief Free a QueryFrame, and all the memory it owns.
7277         @param frame Pointer to the QueryFrame to be freed.
7278 */
7279 static void free_query_frame( QueryFrame* frame ) {
7280         // Sanity check
7281         if( ! frame )
7282                 return;
7283
7284         clear_class_info( &frame->core );
7285
7286         // Free the join list
7287         ClassInfo* temp;
7288         ClassInfo* info = frame->join_list;
7289         while( info ) {
7290                 temp = info->next;
7291                 free_class_info( info );
7292                 info = temp;
7293         }
7294
7295         frame->join_list = NULL;
7296         frame->next = NULL;
7297
7298         // If the frame is a static instance, just mark it as unused
7299         int i;
7300         for( i = 0; i < STATIC_FRAME_COUNT; ++i ) {
7301                 if( frame == static_frame + i ) {
7302                         static_frame[ i ].in_use = 0;
7303                         return;
7304                 }
7305         }
7306
7307         // Otherwise it must have been malloc'd, so free it
7308
7309         free( frame );
7310 }
7311
7312 /**
7313         @brief Search a given QueryFrame for a specified alias.
7314         @param frame Pointer to the QueryFrame to be searched.
7315         @param target The alias for which to search.
7316         @return Pointer to the ClassInfo for the specified alias, if found; otherwise NULL.
7317 */
7318 static ClassInfo* search_alias_in_frame( QueryFrame* frame, const char* target ) {
7319         if( ! frame || ! target ) {
7320                 return NULL;
7321         }
7322
7323         ClassInfo* found_class = NULL;
7324
7325         if( !strcmp( target, frame->core.alias ) )
7326                 return &(frame->core);
7327         else {
7328                 ClassInfo* curr_class = frame->join_list;
7329                 while( curr_class ) {
7330                         if( strcmp( target, curr_class->alias ) )
7331                                 curr_class = curr_class->next;
7332                         else {
7333                                 found_class = curr_class;
7334                                 break;
7335                         }
7336                 }
7337         }
7338
7339         return found_class;
7340 }
7341
7342 /**
7343         @brief Push a new (blank) QueryFrame onto the stack.
7344 */
7345 static void push_query_frame( void ) {
7346         QueryFrame* frame = allocate_frame();
7347         frame->join_list = NULL;
7348         frame->next = curr_query;
7349
7350         // Initialize the ClassInfo for the core class
7351         ClassInfo* core = &frame->core;
7352         core->alias = core->class_name = core->source_def = NULL;
7353         core->class_def = core->fields = core->links = NULL;
7354
7355         curr_query = frame;
7356 }
7357
7358 /**
7359         @brief Pop a QueryFrame off the stack and destroy it.
7360 */
7361 static void pop_query_frame( void ) {
7362         // Sanity check
7363         if( ! curr_query )
7364                 return;
7365
7366         QueryFrame* popped = curr_query;
7367         curr_query = popped->next;
7368
7369         free_query_frame( popped );
7370 }
7371
7372 /**
7373         @brief Populate the ClassInfo for the core class.
7374         @param alias Alias for the core class.  If it is NULL or an empty string, we use the
7375         class name as an alias.
7376         @param class_name Name of the core class.
7377         @return Zero if successful, or 1 if not.
7378
7379         Populate the ClassInfo of the core class with copies of the alias and class name, and
7380         with pointers to the relevant portions of the IDL for the core class.
7381 */
7382 static int add_query_core( const char* alias, const char* class_name ) {
7383
7384         // Sanity checks
7385         if( ! curr_query ) {
7386                 osrfLogError( OSRF_LOG_MARK,
7387                                           "%s ERROR: No QueryFrame available for class %s", modulename, class_name );
7388                 return 1;
7389         } else if( curr_query->core.alias ) {
7390                 osrfLogError( OSRF_LOG_MARK,
7391                                           "%s ERROR: Core class %s already populated as %s",
7392                                           modulename, curr_query->core.class_name, curr_query->core.alias );
7393                 return 1;
7394         }
7395
7396         build_class_info( &curr_query->core, alias, class_name );
7397         if( curr_query->core.alias )
7398                 return 0;
7399         else {
7400                 osrfLogError( OSRF_LOG_MARK,
7401                                           "%s ERROR: Unable to look up core class %s", modulename, class_name );
7402                 return 1;
7403         }
7404 }
7405
7406 /**
7407         @brief Search the current QueryFrame for a specified alias.
7408         @param target The alias for which to search.
7409         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7410 */
7411 static inline ClassInfo* search_alias( const char* target ) {
7412         return search_alias_in_frame( curr_query, target );
7413 }
7414
7415 /**
7416         @brief Search all levels of query for a specified alias, starting with the current query.
7417         @param target The alias for which to search.
7418         @return A pointer to the corresponding ClassInfo, if found; otherwise NULL.
7419 */
7420 static ClassInfo* search_all_alias( const char* target ) {
7421         ClassInfo* found_class = NULL;
7422         QueryFrame* curr_frame = curr_query;
7423
7424         while( curr_frame ) {
7425                 if(( found_class = search_alias_in_frame( curr_frame, target ) ))
7426                         break;
7427                 else
7428                         curr_frame = curr_frame->next;
7429         }
7430
7431         return found_class;
7432 }
7433
7434 /**
7435         @brief Add a class to the list of classes joined to the current query.
7436         @param alias Alias of the class to be added.  If it is NULL or an empty string, we use
7437         the class name as an alias.
7438         @param classname The name of the class to be added.
7439         @return A pointer to the ClassInfo for the added class, if successful; otherwise NULL.
7440 */
7441 static ClassInfo* add_joined_class( const char* alias, const char* classname ) {
7442
7443         if( ! classname || ! *classname ) {    // sanity check
7444                 osrfLogError( OSRF_LOG_MARK, "Can't join a class with no class name" );
7445                 return NULL;
7446         }
7447
7448         if( ! alias )
7449                 alias = classname;
7450
7451         const ClassInfo* conflict = search_alias( alias );
7452         if( conflict ) {
7453                 osrfLogError( OSRF_LOG_MARK,
7454                                           "%s ERROR: Table alias \"%s\" conflicts with class \"%s\"",
7455                                           modulename, alias, conflict->class_name );
7456                 return NULL;
7457         }
7458
7459         ClassInfo* info = allocate_class_info();
7460
7461         if( build_class_info( info, alias, classname ) ) {
7462                 free_class_info( info );
7463                 return NULL;
7464         }
7465
7466         // Add the new ClassInfo to the join list of the current QueryFrame
7467         info->next = curr_query->join_list;
7468         curr_query->join_list = info;
7469
7470         return info;
7471 }
7472
7473 /**
7474         @brief Destroy all nodes on the query stack.
7475 */
7476 static void clear_query_stack( void ) {
7477         while( curr_query )
7478                 pop_query_frame();
7479 }
7480
7481 /**
7482         @brief Implement the set_audit_info method.
7483         @param ctx Pointer to the method context.
7484         @return Zero if successful, or -1 if not.
7485
7486         Issue a SAVEPOINT to the database server.
7487
7488         Method parameters:
7489         - authkey
7490         - user id (int)
7491         - workstation id (int)
7492
7493         If user id is not provided the authkey will be used.
7494         For PCRUD the authkey is always used, even if a user is provided.
7495 */
7496 int setAuditInfo( osrfMethodContext* ctx ) {
7497         if(osrfMethodVerifyContext( ctx )) {
7498                 osrfLogError( OSRF_LOG_MARK,  "Invalid method context" );
7499                 return -1;
7500         }
7501
7502         // Get the user id from the parameters
7503         const char* user_id = jsonObjectGetString( jsonObjectGetIndex(ctx->params, 1) );
7504
7505         if( enforce_pcrud || !user_id ) {
7506                 timeout_needs_resetting = 1;
7507                 const jsonObject* user = verifyUserPCRUD( ctx );
7508                 if( !user )
7509                         return -1;
7510                 osrfAppRespondComplete( ctx, NULL );
7511                 return 0;
7512         }
7513
7514         // Not PCRUD and have a user_id?
7515         int result = writeAuditInfo( ctx, user_id, jsonObjectGetString( jsonObjectGetIndex(ctx->params, 2) ) );
7516         osrfAppRespondComplete( ctx, NULL );
7517         return result;
7518 }
7519
7520 /**
7521         @brief Save a audit info
7522         @param ctx Pointer to the method context.
7523         @param user_id User ID to write as a string
7524         @param ws_id Workstation ID to write as a string
7525 */
7526 int writeAuditInfo( osrfMethodContext* ctx, const char* user_id, const char* ws_id) {
7527         if( ctx && ctx->session ) {
7528                 osrfAppSession* session = ctx->session;
7529
7530                 osrfHash* cache = session->userData;
7531
7532                 // If the session doesn't already have a hash, create one.  Make sure
7533                 // that the application session frees the hash when it terminates.
7534                 if( NULL == cache ) {
7535                         session->userData = cache = osrfNewHash();
7536                         osrfHashSetCallback( cache, &sessionDataFree );
7537                         ctx->session->userDataFree = &userDataFree;
7538                 }
7539
7540                 dbi_result result = dbi_conn_queryf( writehandle, "SELECT auditor.set_audit_info( %s, %s );", user_id, ws_id ? ws_id : "NULL" );
7541                 if( !result ) {
7542                         osrfLogWarning( OSRF_LOG_MARK, "BAD RESULT" );
7543                         const char* msg;
7544                         int errnum = dbi_conn_error( writehandle, &msg );
7545                         osrfLogError(
7546                                 OSRF_LOG_MARK,
7547                                 "%s: Error setting auditor information: %d %s",
7548                                 modulename,
7549                                 errnum,
7550                                 msg ? msg : "(No description available)"
7551                         );
7552                         osrfAppSessionStatus( ctx->session, OSRF_STATUS_INTERNALSERVERERROR,
7553                                 "osrfMethodException", ctx->request, "Error setting auditor info" );
7554                         if( !oilsIsDBConnected( writehandle ))
7555                                 osrfAppSessionPanic( ctx->session );
7556                         return -1;
7557                 } else {
7558                         dbi_result_free( result );
7559                 }
7560         }
7561         return 0;
7562 }
7563
7564 /**
7565         @brief Remove all but safe character from savepoint name
7566         @param sp User-supplied savepoint name
7567         @return sanitized savepoint name, or NULL
7568
7569     The caller is expected to free the returned string.  Note that
7570     this function exists only because we can't use PQescapeLiteral
7571     without either forking libdbi or abandoning it.
7572 */
7573 static char* _sanitize_savepoint_name( const char* sp ) {
7574
7575         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_";
7576
7577         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7578         // and the default value of NAMEDATALEN is 64; that should be long enough
7579         // for our purposes, and it's unlikely that anyone is going to recompile
7580         // PostgreSQL to have a smaller value, so cap the identifier name
7581         // accordingly to avoid the remote chance that someone manages to pass in a
7582         // 12GB savepoint name
7583         const int MAX_LITERAL_NAMELEN = 63;
7584         int len = 0;
7585         len = strlen( sp );
7586         if (len > MAX_LITERAL_NAMELEN) {
7587                 len = MAX_LITERAL_NAMELEN;
7588         }
7589
7590         char* safeSpName = safe_malloc( len + 1 );
7591         int i = 0;
7592         int j;
7593         char* found;
7594         for (j = 0; j < len; j++) {
7595         found = strchr(safe_chars, sp[j]);
7596                 if (found) {
7597                         safeSpName[ i++ ] = found[0];
7598                 }
7599         }
7600         safeSpName[ i ] = '\0';
7601         return safeSpName;
7602 }
7603
7604 /**
7605         @brief Remove all but safe character from TZ name
7606         @param tz User-supplied TZ name
7607         @return sanitized TZ name, or NULL
7608
7609     The caller is expected to free the returned string.  Note that
7610     this function exists only because we can't use PQescapeLiteral
7611     without either forking libdbi or abandoning it.
7612 */
7613 static char* _sanitize_tz_name( const char* tz ) {
7614
7615         if (NULL == tz) return NULL;
7616
7617         const char* safe_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345789_/-+";
7618
7619         // PostgreSQL uses NAMEDATALEN-1 as a max length for identifiers,
7620         // and the default value of NAMEDATALEN is 64; that should be long enough
7621         // for our purposes, and it's unlikely that anyone is going to recompile
7622         // PostgreSQL to have a smaller value, so cap the identifier name
7623         // accordingly to avoid the remote chance that someone manages to pass in a
7624         // 12GB savepoint name
7625         const int MAX_LITERAL_NAMELEN = 63;
7626         int len = 0;
7627         len = strlen( tz );
7628         if (len > MAX_LITERAL_NAMELEN) {
7629                 len = MAX_LITERAL_NAMELEN;
7630         }
7631
7632         char* safeSpName = safe_malloc( len + 1 );
7633         int i = 0;
7634         int j;
7635         char* found;
7636         for (j = 0; j < len; j++) {
7637         found = strchr(safe_chars, tz[j]);
7638                 if (found) {
7639                         safeSpName[ i++ ] = found[0];
7640                 }
7641         }
7642         safeSpName[ i ] = '\0';
7643         return safeSpName;
7644 }
7645
7646 /*@}*/